You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2013/02/10 07:04:01 UTC
[11/50] [abbrv] git commit: BIGTOP-798. introduce a fatjar collection
of all the Bigtop integration tests and all their dependencies
BIGTOP-798. introduce a fatjar collection of all the Bigtop integration tests and all their dependencies
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/a6ca5acd
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/a6ca5acd
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/a6ca5acd
Branch: refs/heads/RCs
Commit: a6ca5acd1e889d8e47521778f08732dbd9cc3db5
Parents: aa7ba33
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Thu Nov 29 14:55:57 2012 -0800
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Fri Nov 30 16:31:33 2012 -0800
----------------------------------------------------------------------
bigtop-tests/test-artifacts/fatjar/pom.xml | 132 +++++++
.../fatjar/src/main/resources/log4j.properties | 20 ++
bigtop-tests/test-artifacts/pom.xml | 1 +
bigtop-tests/test-artifacts/sqoop-smokes/pom.xml | 34 --
.../org/apache/itest/sqoop/TestSqoopExport.groovy | 164 ---------
.../org/apache/itest/sqoop/TestSqoopImport.groovy | 264 ---------------
.../main/resources/mysql-files/mysql-create-db.sql | 39 ---
.../resources/mysql-files/mysql-create-tables.sql | 54 ---
.../resources/mysql-files/mysql-insert-data.sql | 57 ---
.../resources/mysql-files/sqoop-all-tables.out | 2 -
.../main/resources/mysql-files/sqoop-append.out | 20 --
.../main/resources/mysql-files/sqoop-columns.out | 10 -
.../mysql-files/sqoop-null-non-string.out | 3 -
.../resources/mysql-files/sqoop-null-string.out | 3 -
.../src/main/resources/mysql-files/sqoop-query.out | 2 -
.../resources/mysql-files/sqoop-t_bool-export.out | 1 -
.../main/resources/mysql-files/sqoop-t_bool.out | 1 -
.../resources/mysql-files/sqoop-t_date-export.out | 1 -
.../main/resources/mysql-files/sqoop-t_date.out | 1 -
.../src/main/resources/mysql-files/sqoop-t_fp.out | 1 -
.../src/main/resources/mysql-files/sqoop-t_int.out | 1 -
.../main/resources/mysql-files/sqoop-t_string.out | 1 -
.../main/resources/mysql-files/sqoop-testtable.out | 10 -
.../resources/mysql-files/sqoop-where-clause.out | 4 -
bigtop-tests/test-artifacts/sqoop/pom.xml | 6 +-
.../bigtop/itest/sqoop/TestSqoopExport.groovy | 164 +++++++++
.../bigtop/itest/sqoop/TestSqoopImport.groovy | 264 +++++++++++++++
.../main/resources/mysql-files/mysql-create-db.sql | 39 +++
.../resources/mysql-files/mysql-create-tables.sql | 54 +++
.../resources/mysql-files/mysql-insert-data.sql | 57 +++
.../resources/mysql-files/sqoop-all-tables.out | 2 +
.../main/resources/mysql-files/sqoop-append.out | 20 ++
.../main/resources/mysql-files/sqoop-columns.out | 10 +
.../mysql-files/sqoop-null-non-string.out | 3 +
.../resources/mysql-files/sqoop-null-string.out | 3 +
.../src/main/resources/mysql-files/sqoop-query.out | 2 +
.../resources/mysql-files/sqoop-t_bool-export.out | 1 +
.../main/resources/mysql-files/sqoop-t_bool.out | 1 +
.../resources/mysql-files/sqoop-t_date-export.out | 1 +
.../main/resources/mysql-files/sqoop-t_date.out | 1 +
.../src/main/resources/mysql-files/sqoop-t_fp.out | 1 +
.../src/main/resources/mysql-files/sqoop-t_int.out | 1 +
.../main/resources/mysql-files/sqoop-t_string.out | 1 +
.../main/resources/mysql-files/sqoop-testtable.out | 10 +
.../resources/mysql-files/sqoop-where-clause.out | 4 +
45 files changed, 795 insertions(+), 676 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/fatjar/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/fatjar/pom.xml b/bigtop-tests/test-artifacts/fatjar/pom.xml
new file mode 100644
index 0000000..4ca88b6
--- /dev/null
+++ b/bigtop-tests/test-artifacts/fatjar/pom.xml
@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>bigtop-smokes</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>fat-smoke</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <name>fatjar</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>crunch-smoke</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>flume-smoke</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>giraph-smoke</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>hadoop-smoke</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>hbase-smoke</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>hive-smoke</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>hue-smoke</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>mahout-smoke</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>oozie-smoke</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>solr-smoke</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>sqoop-smoke</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>package-smoke</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ <optional>true</optional>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>assembly</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/fatjar/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/fatjar/src/main/resources/log4j.properties b/bigtop-tests/test-artifacts/fatjar/src/main/resources/log4j.properties
new file mode 100644
index 0000000..ac7a39d
--- /dev/null
+++ b/bigtop-tests/test-artifacts/fatjar/src/main/resources/log4j.properties
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+log4j.rootLogger=${org.apache.bigtop.itest.log4j.level},console
+log4j.threshhold=ALL
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/pom.xml b/bigtop-tests/test-artifacts/pom.xml
index 9eaab89..fcec907 100644
--- a/bigtop-tests/test-artifacts/pom.xml
+++ b/bigtop-tests/test-artifacts/pom.xml
@@ -44,6 +44,7 @@
<module>giraph</module>
<module>hue</module>
<module>solr</module>
+ <module>fatjar</module>
</modules>
<properties>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/pom.xml b/bigtop-tests/test-artifacts/sqoop-smokes/pom.xml
deleted file mode 100644
index e50884d..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/pom.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.bigtop.itest</groupId>
- <artifactId>bigtop-smokes</artifactId>
- <version>0.5.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <groupId>org.apache.bigtop.itest</groupId>
- <artifactId>sqoopsmokes</artifactId>
- <version>0.5.0-SNAPSHOT</version>
- <name>sqoopsmokes</name>
-</project>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/groovy/org/apache/itest/sqoop/TestSqoopExport.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/groovy/org/apache/itest/sqoop/TestSqoopExport.groovy b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/groovy/org/apache/itest/sqoop/TestSqoopExport.groovy
deleted file mode 100644
index b9c561c..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/groovy/org/apache/itest/sqoop/TestSqoopExport.groovy
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.itest.sqoop;
-
-import static org.junit.Assert.assertEquals
-import static org.junit.Assert.assertNotNull
-import static org.junit.Assert.assertTrue
-import org.junit.AfterClass
-import org.junit.BeforeClass
-import org.junit.Test
-
-import org.apache.bigtop.itest.JarContent
-import org.apache.bigtop.itest.shell.Shell
-
-class TestSqoopExport {
- private static String mysql_user =
- System.getenv("MYSQL_USER");
- private static String mysql_password =
- System.getenv("MYSQL_PASSWORD");
- private static final String MYSQL_USER =
- (mysql_user == null) ? "root" : mysql_user;
- private static final String MYSQL_PASSWORD =
- (mysql_password == null) ? "" : mysql_password;
- private static final String MYSQL_HOST = System.getProperty("mysql.host", "localhost");
- private static final String HADOOP_HOME =
- System.getenv('HADOOP_HOME');
- private static String streaming_home = System.getenv('STREAMING_HOME');
- private static final String STREAMING_HOME =
- (streaming_home == null) ? HADOOP_HOME + "/contrib/streaming" :
- streaming_home;
- private static final String SQOOP_HOME =
- System.getenv("SQOOP_HOME");
- static {
- assertNotNull("HADOOP_HOME is not set", HADOOP_HOME);
- assertNotNull("SQOOP_HOME is not set", SQOOP_HOME);
- assertNotNull("mysql connector jar is required to be present in $SQOOP_HOME/lib",
- JarContent.getJarName("$SQOOP_HOME/lib", "mysql-connector-java.*.jar"));
- }
- private static String sqoop_jar =
- JarContent.getJarName(SQOOP_HOME, "sqoop-1.*.jar");
- private static String streaming_jar =
- JarContent.getJarName(STREAMING_HOME, "hadoop.*streaming.*.jar");
- static {
- assertNotNull("Can't find sqoop.jar", sqoop_jar);
- assertNotNull("Can't find hadoop-streaming.jar", streaming_jar);
- }
- private static final String SQOOP_JAR = SQOOP_HOME + "/" + sqoop_jar;
- private static final String STREAMING_JAR = STREAMING_HOME + "/" + streaming_jar;
- private static final String MYSQL_COMMAND =
- "mysql --user=$MYSQL_USER" +
- (("".equals(MYSQL_PASSWORD)) ? "" : " --password=$MYSQL_PASSWORD");
- private static final String MYSQL_DBNAME = System.getProperty("mysql.dbname", "mysqltestdb");
- private static final String SQOOP_CONNECTION =
- "--connect jdbc:mysql://$MYSQL_HOST/$MYSQL_DBNAME --username=$MYSQL_USER" +
- (("".equals(MYSQL_PASSWORD)) ? "" : " --password=$MYSQL_PASSWORD");
- static {
- System.out.println("SQOOP_CONNECTION string is " + SQOOP_CONNECTION );
- }
- private static final String DATA_DIR = System.getProperty("data.dir", "mysql-files");
- private static final String INPUT = System.getProperty("input.dir", "input-dir");
- private static Shell sh = new Shell("/bin/bash -s");
-
- @BeforeClass
- static void setUp() {
- sh.exec("hadoop fs -test -e $INPUT");
- if (sh.getRet() == 0) {
- sh.exec("hadoop fs -rmr -skipTrash $INPUT");
- assertTrue("Deletion of previous $INPUT from HDFS failed",
- sh.getRet() == 0);
- }
- sh.exec("hadoop fs -mkdir $INPUT");
- assertTrue("Could not create $INPUT directory", sh.getRet() == 0);
- // unpack resource
- JarContent.unpackJarContainer(TestSqoopExport.class, '.' , null)
- // upload data to HDFS
- sh.exec("hadoop fs -put $DATA_DIR/sqoop-testtable.out input-dir/testtable/part-m-00000");
- sh.exec("hadoop fs -put $DATA_DIR/sqoop-t_bool.out $INPUT/t_bool/part-m-00000");
- sh.exec("hadoop fs -put $DATA_DIR/sqoop-t_date-export.out $INPUT/t_date/part-m-00000");
- sh.exec("hadoop fs -put $DATA_DIR/sqoop-t_string.out $INPUT/t_string/part-m-00000");
- sh.exec("hadoop fs -put $DATA_DIR/sqoop-t_fp.out $INPUT/t_fp/part-m-00000");
- sh.exec("hadoop fs -put $DATA_DIR/sqoop-t_int.out $INPUT/t_int/part-m-00000");
-
- //create db
- sh.exec("cat $DATA_DIR/mysql-create-db.sql | $MYSQL_COMMAND");
- //create tables
- sh.exec("cat $DATA_DIR/mysql-create-tables.sql | $MYSQL_COMMAND");
- }
-
- @AfterClass
- static void tearDown() {
- if ('YES'.equals(System.getProperty('delete.testdata','no').toUpperCase())) {
- sh.exec("hadoop fs -test -e $INPUT");
- if (sh.getRet() == 0) {
- // sh.exec("hadoop fs -rmr -skipTrash $INPUT");
- assertTrue("Deletion of $INPUT from HDFS failed",
- sh.getRet() == 0);
- }
- }
- }
-
- @Test
- public void testBooleanExport() {
- sh.exec("sqoop export $SQOOP_CONNECTION --table t_bool --export-dir $INPUT/t_bool");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("echo 'use mysqltestdb;select * from t_bool' | $MYSQL_COMMAND --skip-column-names | sed 's/\t/,/g' > t_bool.out");
- assertEquals("sqoop export did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-t_bool-export.out t_bool.out").getRet());
- }
-
-
- @Test
- public void testIntegerExport() {
- sh.exec("sqoop export $SQOOP_CONNECTION --table t_int --export-dir $INPUT/t_int");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("echo 'use mysqltestdb;select * from t_int' | $MYSQL_COMMAND --skip-column-names | sed 's/\t/,/g' > t_int.out");
- assertEquals("sqoop export did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-t_int.out t_int.out").getRet());
- }
-
- @Test
- public void testFixedPointFloatingPointExport() {
- sh.exec("sqoop export $SQOOP_CONNECTION --table t_fp --export-dir $INPUT/t_fp");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("echo 'use mysqltestdb;select * from t_fp' | $MYSQL_COMMAND --skip-column-names | sed 's/\t/,/g' > t_fp.out");
- assertEquals("sqoop export did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-t_fp.out t_fp.out").getRet());
- }
-
- @Test
- public void testDateTimeExport() {
- sh.exec("sqoop export $SQOOP_CONNECTION --table t_date --export-dir $INPUT/t_date");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("echo 'use mysqltestdb;select * from t_date' | $MYSQL_COMMAND --skip-column-names | sed 's/\t/,/g' > t_date.out");
- assertEquals("sqoop export did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-t_date.out t_date.out").getRet());
- }
-
- @Test
- public void testStringExport() {
- sh.exec("sqoop export $SQOOP_CONNECTION --table t_string --export-dir $INPUT/t_string");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("echo 'use mysqltestdb;select * from t_string' | $MYSQL_COMMAND --skip-column-names | sed 's/\t/,/g' > t_string.out");
- assertEquals("sqoop export did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-t_string.out t_string.out").getRet());
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/groovy/org/apache/itest/sqoop/TestSqoopImport.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/groovy/org/apache/itest/sqoop/TestSqoopImport.groovy b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/groovy/org/apache/itest/sqoop/TestSqoopImport.groovy
deleted file mode 100644
index 5845ee2..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/groovy/org/apache/itest/sqoop/TestSqoopImport.groovy
+++ /dev/null
@@ -1,264 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.itest.sqoop;
-
-import static org.junit.Assert.assertEquals
-import static org.junit.Assert.assertNotNull
-import static org.junit.Assert.assertTrue
-import org.junit.AfterClass
-import org.junit.BeforeClass
-import org.junit.Test
-
-import org.apache.bigtop.itest.JarContent
-import org.apache.bigtop.itest.shell.Shell
-
-class TestSqoopImport {
- private static String mysql_user =
- System.getenv("MYSQL_USER");
- private static String mysql_password =
- System.getenv("MYSQL_PASSWORD");
- private static final String MYSQL_USER =
- (mysql_user == null) ? "root" : mysql_user;
- private static final String MYSQL_PASSWORD =
- (mysql_password == null) ? "" : mysql_password;
- private static final String MYSQL_HOST = System.getProperty("mysql.host", "localhost");
- private static final String HADOOP_HOME =
- System.getenv('HADOOP_HOME');
- private static String streaming_home = System.getenv('STREAMING_HOME');
- private static final String STREAMING_HOME =
- (streaming_home == null) ? HADOOP_HOME + "/contrib/streaming" :
- streaming_home;
- private static final String SQOOP_HOME =
- System.getenv("SQOOP_HOME");
- static {
- assertNotNull("HADOOP_HOME is not set", HADOOP_HOME);
- assertNotNull("SQOOP_HOME is not set", SQOOP_HOME);
- assertNotNull("mysql connector jar is required to be present in $SQOOP_HOME/lib",
- JarContent.getJarName("$SQOOP_HOME/lib", "mysql-connector-java.*.jar"));
- }
- private static String sqoop_jar =
- JarContent.getJarName(SQOOP_HOME, "sqoop-1.*.jar");
- private static String streaming_jar =
- JarContent.getJarName(STREAMING_HOME, "hadoop.*streaming.*.jar");
- static {
- assertNotNull("Can't find sqoop.jar", sqoop_jar);
- assertNotNull("Can't find hadoop-streaming.jar", streaming_jar);
- }
- private static final String SQOOP_JAR = SQOOP_HOME + "/" + sqoop_jar;
- private static final String STREAMING_JAR = STREAMING_HOME + "/" + streaming_jar;
- private static final String MYSQL_COMMAND =
- "mysql --user=$MYSQL_USER" +
- (("".equals(MYSQL_PASSWORD)) ? "" : " --password=$MYSQL_PASSWORD");
- private static final String MYSQL_DBNAME = System.getProperty("mysql.dbname", "mysqltestdb");
- private static final String SQOOP_CONNECTION =
- "--connect jdbc:mysql://$MYSQL_HOST/$MYSQL_DBNAME --username=$MYSQL_USER" +
- (("".equals(MYSQL_PASSWORD)) ? "" : " --password=$MYSQL_PASSWORD");
- static {
- System.out.println("SQOOP_CONNECTION string is " + SQOOP_CONNECTION );
- }
- private static final String DATA_DIR = System.getProperty("data.dir", "mysql-files");
- private static final String OUTPUT = System.getProperty("output.dir", "output-dir");
- private static Shell sh = new Shell("/bin/bash -s");
-
- @BeforeClass
- static void setUp() {
- sh.exec("hadoop fs -test -e $OUTPUT");
- if (sh.getRet() == 0) {
- sh.exec("hadoop fs -rmr -skipTrash $OUTPUT");
- assertTrue("Deletion of previous $OUTPUT from HDFS failed",
- sh.getRet() == 0);
- }
- sh.exec("hadoop fs -mkdir $OUTPUT");
- assertTrue("Could not create $OUTPUT directory", sh.getRet() == 0);
- // unpack resource
- JarContent.unpackJarContainer(TestSqoopImport.class, '.' , null)
- // create the database
- sh.exec("cat $DATA_DIR/mysql-create-db.sql | $MYSQL_COMMAND");
- //create tables
- sh.exec("cat $DATA_DIR/mysql-create-tables.sql | $MYSQL_COMMAND");
- //populate data
- sh.exec("cat $DATA_DIR/mysql-insert-data.sql | $MYSQL_COMMAND");
- }
-
- @AfterClass
- static void tearDown() {
- if ('YES'.equals(System.getProperty('delete.testdata','no').toUpperCase())) {
- sh.exec("hadoop fs -test -e $OUTPUT");
- if (sh.getRet() == 0) {
- sh.exec("hadoop fs -rmr -skipTrash $OUTPUT");
- assertTrue("Deletion of $OUTPUT from HDFS failed",
- sh.getRet() == 0);
- }
- }
- }
-
- @Test
- public void testBooleanImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table t_bool --target-dir $OUTPUT/t_bool");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/t_bool/part-* > t_bool.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-t_bool.out t_bool.out").getRet());
- }
-
-
- @Test
- public void testIntegerImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table t_int --target-dir $OUTPUT/t_int");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/t_int/part-* > t_int.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-t_int.out t_int.out").getRet());
- }
-
- @Test
- public void testFixedPointFloatingPointImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table t_fp --target-dir $OUTPUT/t_fp");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/t_fp/part-* > t_fp.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-t_fp.out t_fp.out").getRet());
- }
-
- @Test
- public void testDateTimeImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table t_date --target-dir $OUTPUT/t_date");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/t_date/part-* > t_date.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-t_date.out t_date.out").getRet());
- }
-
- @Test
- public void testStringImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table t_string --target-dir $OUTPUT/t_string");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/t_string/part-* > t_string.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-t_string.out t_string.out").getRet());
- }
-
- @Test
- public void testAppendImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --target-dir $OUTPUT/append");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- //import again with append
- sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --append --target-dir $OUTPUT/append");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/append/part-* > append.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-append.out append.out").getRet());
- }
-
- @Test
- public void testColumnsImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --columns id,fname --target-dir $OUTPUT/columns");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/columns/part-* > columns.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-columns.out columns.out").getRet());
- }
-
- @Test
- public void testDirectImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --direct --target-dir $OUTPUT/direct");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/direct/part-* > direct.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-testtable.out direct.out").getRet());
- }
-
- @Test
- public void testNumMappersImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --num-mappers 1 --target-dir $OUTPUT/num-mappers");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/num-mappers/part-*0 > num-mappers.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-testtable.out num-mappers.out").getRet());
- }
-
- @Test
- public void testQueryImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --query 'select t1.id as id, t2.fname as fname from testtable as t1 join testtable2 as t2 on (t1.id = t2.id ) where t1.id < 3 AND \$CONDITIONS' --split-by t1.id --target-dir $OUTPUT/query");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/query/part-* > query.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-query.out query.out").getRet());
- }
-
- @Test
- public void testSplityByImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --split-by fname --target-dir $OUTPUT/split-by");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/split-by/part-* > split-by.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-testtable.out split-by.out").getRet());
- }
-
- @Test
- public void testWarehouseDirImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --warehouse-dir $OUTPUT/warehouse-dir");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/warehouse-dir/testtable/part-* > warehouse-dir.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-testtable.out warehouse-dir.out").getRet());
- }
-
- @Test
- public void testWhereClauseImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --where \"id < 5\" --target-dir $OUTPUT/where-clause");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/where-clause/part-* > where-clause.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-where-clause.out where-clause.out").getRet());
- }
-
- @Test
- public void testNullStringImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table testnullvalues --null-string mynullstring --target-dir $OUTPUT/null-string");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/null-string/part-* > null-string.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-null-string.out null-string.out").getRet());
- }
-
- @Test
- public void testNullNonStringImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table testnullvalues --null-non-string 10 --target-dir $OUTPUT/non-null-string");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/non-null-string/part-* > non-null-string.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-null-non-string.out non-null-string.out").getRet());
- }
-
- //database name is hardcoded here
- @Test
- public void testImportAllTables() {
- String SQOOP_CONNECTION_IMPORT_ALL =
- "--connect jdbc:mysql://$MYSQL_HOST/mysqltestdb2 --username=$MYSQL_USER" +
- (("".equals(MYSQL_PASSWORD)) ? "" : " --password=$MYSQL_PASSWORD");
-
- sh.exec("sqoop import-all-tables $SQOOP_CONNECTION_IMPORT_ALL --warehouse-dir $OUTPUT/alltables");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/alltables/testtable*/part-* > all-tables.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-all-tables.out all-tables.out").getRet());
- }
-}
-
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/mysql-create-db.sql
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/mysql-create-db.sql b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/mysql-create-db.sql
deleted file mode 100644
index 7746241..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/mysql-create-db.sql
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-drop database if exists mysqltestdb;
-
-create database mysqltestdb;
-
-use mysqltestdb;
-
-grant all privileges on mysqltestdb.* to 'root'@'%';
-
-flush privileges;
-
-/* create database2 for import-all */
-
-drop database if exists mysqltestdb2;
-
-create database mysqltestdb2;
-
-use mysqltestdb2;
-
-grant all privileges on mysqltestdb2.* to 'root'@'%';
-
-flush privileges;
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/mysql-create-tables.sql
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/mysql-create-tables.sql b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/mysql-create-tables.sql
deleted file mode 100644
index 5a4a7ce..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/mysql-create-tables.sql
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/* functional test db tables */
-use mysqltestdb;
-
-drop table if exists t_bool;
-create table t_bool ( pri int not null auto_increment, a boolean, primary key (pri));
-
-drop table if exists t_int;
-create table t_int ( pri int not null auto_increment,a tinyint, b smallint, c mediumint , d int, e integer , f bigint,primary key (pri));
-
-drop table if exists t_fp;
-create table t_fp (pri int not null auto_increment, a decimal (6,4), b float (6,4), c double precision (6,4),d float (4),primary key (pri));
-
-drop table if exists t_date;
-create table t_date (pri int not null auto_increment,a date , c timestamp, d time , e year(4),primary key (pri));
-
-drop table if exists t_string;
-create table t_string (pri int not null auto_increment, a char (5), b varchar (5), c text , d tinytext, e mediumtext, f mediumtext, g longtext, h enum ('A','B'),primary key (pri));
-
-drop table if exists testtable;
-create table testtable ( id int , fname varchar (20), lname varchar(20) , primary key (id));
-
-drop table if exists testtable2;
-create table testtable2 ( id int , fname varchar (20), lname varchar(20) , primary key (id));
-
-drop table if exists testnullvalues;
-create table testnullvalues ( id int , a int , b varchar(20) , primary key (id));
-
-/* import-all test db-tables */
-
-use mysqltestdb2;
-
-drop table if exists testtable;
-create table testtable ( id int , fname varchar (20), lname varchar(20) , primary key (id));
-
-drop table if exists testtable2;
-create table testtable2 ( id int , fname varchar (20), lname varchar(20) , primary key (id));
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/mysql-insert-data.sql
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/mysql-insert-data.sql b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/mysql-insert-data.sql
deleted file mode 100644
index c8aef22..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/mysql-insert-data.sql
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-use mysqltestdb;
-
-insert into t_int (a,b,c,d,e,f) values (127,32767,8388607,2147483647,-2147483648,9223372036854775807);
-
-insert into t_bool (a) values (1);
-
-insert into t_fp(a,b,c,d) values (-99.9999,-99.9999,99.9999,99.99);
-
-insert into t_date (a,c,d,e) values ('9999-12-31',19731230153000,'23:59:59','2155');
-
-insert into t_string (a,b,c,d,e,f,g,h) values ('aaa','aaa','0GIQM3CONg5jAUloRoP7f76TJrYkYnto5i76IUDbKuG1dMSP5znIM6Ct3qc1WdRCD9THFjWXBIxOoAqhOgdFQEwICX7fYN4n9P3AOoo3ZrnUBu3rtuwf9Y2HXJvUBgAqs44Ypdg6iv511JQSufMeKvbLIpIbk9saJW82EeTgoWkAS7lhI0HJXmvyDkJEqOc0VxdT0ySR','0GIQM3CONg5jAUloRoP7f76TJrYkYnto5i76IUDbKuG1dMSP5znIM6Ct3qc1WdRCD9THFjWXBIxOoAqhOgdFQEwICX7fYN4n9P3AOoo3ZrnUBu3rtuwf9Y2HXJvUBgAqs44Ypdg6iv511JQSufMeKvbLIpIbk9saJW82EeTgoWkAS7lhI0HJXmvyDkJEqOc0VxdT0ySR','0GIQM3CONg5jAUloRoP7f76TJrYkYnto5i76IUDbKuG1dMSP5znIM6Ct3qc1WdRCD9THFjWXBIxOoAqhOgdFQEwICX7fYN4n9P3AOoo3ZrnUBu3rtuwf9Y2HXJvUBgAqs44Ypdg6iv511JQSufMeKvbLIpIbk9saJW82EeTgoWkAS7lhI0HJXmvyDkJEqOc0VxdT0ySR','0GIQM3CONg5jAUloRoP7f76TJrYkYnto5i76IUDbKuG1dMSP5znIM6Ct3qc1WdRCD9THFjWXBIxOoAqhOgdFQEwICX7fYN4n9P3AOoo3ZrnUBu3rtuwf9Y2HXJvUBgAqs44Ypdg6iv511JQSufMeKvbLIpIbk9saJW82EeTgoWkAS7lhI0HJXmvyDkJEqOc0VxdT0ySR','0GIQM3CONg5jAUloRoP7f76TJrYkYnto5i76IUDbKuG1dMSP5znIM6Ct3qc1WdRCD9THFjWXBIxOoAqhOgdFQEwICX7fYN4n9P3AOoo3ZrnUBu3rtuwf9Y2HXJvUB
gAqs44Ypdg6iv511JQSufMeKvbLIpIbk9saJW82EeTgoWkAS7lhI0HJXmvyDkJEqOc0VxdT0ySR','A');
-
-insert into testtable values (1,'aaa','aaa');
-insert into testtable values (2,'bbb','bbb');
-insert into testtable values (3,'ccc','ccc');
-insert into testtable values (4,'ddd','ddd');
-insert into testtable values (5,'eee','eee');
-insert into testtable values (6,'fff','fff');
-insert into testtable values (7,'ggg','ggg');
-insert into testtable values (8,'hhh','hhh');
-insert into testtable values (9,'iii','iii');
-insert into testtable values (10,'jjj','jjj');
-
-
-insert into testtable2 values (1,'111','111');
-insert into testtable2 values (2,'222','222');
-insert into testtable2 values (3,'333','333');
-
-
-insert into testnullvalues (id) values (1);
-insert into testnullvalues (id,a) values (2,2);
-insert into testnullvalues (id,b) values (3,'aaa');
-
-/* data for import-all test */
-
-use mysqltestdb2;
-
-insert into testtable values (1,'aaa','aaa');
-insert into testtable2 values (1,'aaa','aaa');
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-all-tables.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-all-tables.out b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-all-tables.out
deleted file mode 100644
index 855e550..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-all-tables.out
+++ /dev/null
@@ -1,2 +0,0 @@
-1,aaa,aaa
-1,aaa,aaa
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-append.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-append.out b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-append.out
deleted file mode 100644
index 98d09c4..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-append.out
+++ /dev/null
@@ -1,20 +0,0 @@
-1,aaa,aaa
-2,bbb,bbb
-3,ccc,ccc
-4,ddd,ddd
-5,eee,eee
-6,fff,fff
-7,ggg,ggg
-8,hhh,hhh
-9,iii,iii
-10,jjj,jjj
-1,aaa,aaa
-2,bbb,bbb
-3,ccc,ccc
-4,ddd,ddd
-5,eee,eee
-6,fff,fff
-7,ggg,ggg
-8,hhh,hhh
-9,iii,iii
-10,jjj,jjj
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-columns.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-columns.out b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-columns.out
deleted file mode 100644
index dd3c973..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-columns.out
+++ /dev/null
@@ -1,10 +0,0 @@
-1,aaa
-2,bbb
-3,ccc
-4,ddd
-5,eee
-6,fff
-7,ggg
-8,hhh
-9,iii
-10,jjj
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-null-non-string.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-null-non-string.out b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-null-non-string.out
deleted file mode 100644
index c1ef144..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-null-non-string.out
+++ /dev/null
@@ -1,3 +0,0 @@
-1,10,null
-2,2,null
-3,10,aaa
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-null-string.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-null-string.out b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-null-string.out
deleted file mode 100644
index acddc7b..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-null-string.out
+++ /dev/null
@@ -1,3 +0,0 @@
-1,null,mynullstring
-2,2,mynullstring
-3,null,aaa
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-query.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-query.out b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-query.out
deleted file mode 100644
index 786a05c..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-query.out
+++ /dev/null
@@ -1,2 +0,0 @@
-1,111
-2,222
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_bool-export.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_bool-export.out b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_bool-export.out
deleted file mode 100644
index 9474240..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_bool-export.out
+++ /dev/null
@@ -1 +0,0 @@
-1,1
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_bool.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_bool.out b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_bool.out
deleted file mode 100644
index 5f49b20..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_bool.out
+++ /dev/null
@@ -1 +0,0 @@
-1,true
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_date-export.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_date-export.out b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_date-export.out
deleted file mode 100644
index 992838e..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_date-export.out
+++ /dev/null
@@ -1 +0,0 @@
-1,"9999-12-31","9999-12-31 23:59:59","1973-12-30 15:30:00","23:59:59","2155"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_date.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_date.out b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_date.out
deleted file mode 100644
index 54cf66f..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_date.out
+++ /dev/null
@@ -1 +0,0 @@
-1,9999-12-31,1973-12-30 15:30:00.0,23:59:59,2155
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_fp.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_fp.out b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_fp.out
deleted file mode 100644
index 1ce8d4f..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_fp.out
+++ /dev/null
@@ -1 +0,0 @@
-1,-99.9999,-99.9999,99.9999,99.99
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_int.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_int.out b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_int.out
deleted file mode 100644
index 00a5f1a..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_int.out
+++ /dev/null
@@ -1 +0,0 @@
-1,127,32767,8388607,2147483647,-2147483648,9223372036854775807
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_string.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_string.out b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_string.out
deleted file mode 100644
index d98f0df..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-t_string.out
+++ /dev/null
@@ -1 +0,0 @@
-1,aaa,aaa,0GIQM3CONg5jAUloRoP7f76TJrYkYnto5i76IUDbKuG1dMSP5znIM6Ct3qc1WdRCD9THFjWXBIxOoAqhOgdFQEwICX7fYN4n9P3AOoo3ZrnUBu3rtuwf9Y2HXJvUBgAqs44Ypdg6iv511JQSufMeKvbLIpIbk9saJW82EeTgoWkAS7lhI0HJXmvyDkJEqOc0VxdT0ySR,0GIQM3CONg5jAUloRoP7f76TJrYkYnto5i76IUDbKuG1dMSP5znIM6Ct3qc1WdRCD9THFjWXBIxOoAqhOgdFQEwICX7fYN4n9P3AOoo3ZrnUBu3rtuwf9Y2HXJvUBgAqs44Ypdg6iv511JQSufMeKvbLIpIbk9saJW82EeTgoWkAS7lhI0HJXmvyDkJEqOc0VxdT0ySR,0GIQM3CONg5jAUloRoP7f76TJrYkYnto5i76IUDbKuG1dMSP5znIM6Ct3qc1WdRCD9THFjWXBIxOoAqhOgdFQEwICX7fYN4n9P3AOoo3ZrnUBu3rtuwf9Y2HXJvUBgAqs44Ypdg6iv511JQSufMeKvbLIpIbk9saJW82EeTgoWkAS7lhI0HJXmvyDkJEqOc0VxdT0ySR,0GIQM3CONg5jAUloRoP7f76TJrYkYnto5i76IUDbKuG1dMSP5znIM6Ct3qc1WdRCD9THFjWXBIxOoAqhOgdFQEwICX7fYN4n9P3AOoo3ZrnUBu3rtuwf9Y2HXJvUBgAqs44Ypdg6iv511JQSufMeKvbLIpIbk9saJW82EeTgoWkAS7lhI0HJXmvyDkJEqOc0VxdT0ySR,0GIQM3CONg5jAUloRoP7f76TJrYkYnto5i76IUDbKuG1dMSP5znIM6Ct3qc1WdRCD9THFjWXBIxOoAqhOgdFQEwICX7fYN4n9P3AOoo3ZrnUBu3rtuwf9Y2HXJvUBgAqs44Ypdg6iv511JQSufMeKvbLIpIbk9saJW82EeTgoWkAS7lhI0HJXmv
yDkJEqOc0VxdT0ySR,A
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-testtable.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-testtable.out b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-testtable.out
deleted file mode 100644
index 4f78d6e..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-testtable.out
+++ /dev/null
@@ -1,10 +0,0 @@
-1,aaa,aaa
-2,bbb,bbb
-3,ccc,ccc
-4,ddd,ddd
-5,eee,eee
-6,fff,fff
-7,ggg,ggg
-8,hhh,hhh
-9,iii,iii
-10,jjj,jjj
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-where-clause.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-where-clause.out b/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-where-clause.out
deleted file mode 100644
index 1eeb597..0000000
--- a/bigtop-tests/test-artifacts/sqoop-smokes/src/main/resources/mysql-files/sqoop-where-clause.out
+++ /dev/null
@@ -1,4 +0,0 @@
-1,aaa,aaa
-2,bbb,bbb
-3,ccc,ccc
-4,ddd,ddd
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/pom.xml b/bigtop-tests/test-artifacts/sqoop/pom.xml
index 9b18929..591c9aa 100644
--- a/bigtop-tests/test-artifacts/sqoop/pom.xml
+++ b/bigtop-tests/test-artifacts/sqoop/pom.xml
@@ -15,11 +15,11 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
+
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
-
<parent>
<groupId>org.apache.bigtop.itest</groupId>
<artifactId>bigtop-smokes</artifactId>
@@ -28,7 +28,7 @@
</parent>
<groupId>org.apache.bigtop.itest</groupId>
- <artifactId>sqoop-integration</artifactId>
+ <artifactId>sqoop-smoke</artifactId>
<version>0.5.0-SNAPSHOT</version>
- <name>sqoop-integration</name>
+ <name>sqoopsmokes</name>
</project>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop/src/main/groovy/org/apache/bigtop/itest/sqoop/TestSqoopExport.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/src/main/groovy/org/apache/bigtop/itest/sqoop/TestSqoopExport.groovy b/bigtop-tests/test-artifacts/sqoop/src/main/groovy/org/apache/bigtop/itest/sqoop/TestSqoopExport.groovy
new file mode 100644
index 0000000..b9c561c
--- /dev/null
+++ b/bigtop-tests/test-artifacts/sqoop/src/main/groovy/org/apache/bigtop/itest/sqoop/TestSqoopExport.groovy
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.itest.sqoop;
+
+import static org.junit.Assert.assertEquals
+import static org.junit.Assert.assertNotNull
+import static org.junit.Assert.assertTrue
+import org.junit.AfterClass
+import org.junit.BeforeClass
+import org.junit.Test
+
+import org.apache.bigtop.itest.JarContent
+import org.apache.bigtop.itest.shell.Shell
+
+class TestSqoopExport {
+ private static String mysql_user =
+ System.getenv("MYSQL_USER");
+ private static String mysql_password =
+ System.getenv("MYSQL_PASSWORD");
+ private static final String MYSQL_USER =
+ (mysql_user == null) ? "root" : mysql_user;
+ private static final String MYSQL_PASSWORD =
+ (mysql_password == null) ? "" : mysql_password;
+ private static final String MYSQL_HOST = System.getProperty("mysql.host", "localhost");
+ private static final String HADOOP_HOME =
+ System.getenv('HADOOP_HOME');
+ private static String streaming_home = System.getenv('STREAMING_HOME');
+ private static final String STREAMING_HOME =
+ (streaming_home == null) ? HADOOP_HOME + "/contrib/streaming" :
+ streaming_home;
+ private static final String SQOOP_HOME =
+ System.getenv("SQOOP_HOME");
+ static {
+ assertNotNull("HADOOP_HOME is not set", HADOOP_HOME);
+ assertNotNull("SQOOP_HOME is not set", SQOOP_HOME);
+ assertNotNull("mysql connector jar is required to be present in $SQOOP_HOME/lib",
+ JarContent.getJarName("$SQOOP_HOME/lib", "mysql-connector-java.*.jar"));
+ }
+ private static String sqoop_jar =
+ JarContent.getJarName(SQOOP_HOME, "sqoop-1.*.jar");
+ private static String streaming_jar =
+ JarContent.getJarName(STREAMING_HOME, "hadoop.*streaming.*.jar");
+ static {
+ assertNotNull("Can't find sqoop.jar", sqoop_jar);
+ assertNotNull("Can't find hadoop-streaming.jar", streaming_jar);
+ }
+ private static final String SQOOP_JAR = SQOOP_HOME + "/" + sqoop_jar;
+ private static final String STREAMING_JAR = STREAMING_HOME + "/" + streaming_jar;
+ private static final String MYSQL_COMMAND =
+ "mysql --user=$MYSQL_USER" +
+ (("".equals(MYSQL_PASSWORD)) ? "" : " --password=$MYSQL_PASSWORD");
+ private static final String MYSQL_DBNAME = System.getProperty("mysql.dbname", "mysqltestdb");
+ private static final String SQOOP_CONNECTION =
+ "--connect jdbc:mysql://$MYSQL_HOST/$MYSQL_DBNAME --username=$MYSQL_USER" +
+ (("".equals(MYSQL_PASSWORD)) ? "" : " --password=$MYSQL_PASSWORD");
+ static {
+ System.out.println("SQOOP_CONNECTION string is " + SQOOP_CONNECTION );
+ }
+ private static final String DATA_DIR = System.getProperty("data.dir", "mysql-files");
+ private static final String INPUT = System.getProperty("input.dir", "input-dir");
+ private static Shell sh = new Shell("/bin/bash -s");
+
+ @BeforeClass
+ static void setUp() {
+ sh.exec("hadoop fs -test -e $INPUT");
+ if (sh.getRet() == 0) {
+ sh.exec("hadoop fs -rmr -skipTrash $INPUT");
+ assertTrue("Deletion of previous $INPUT from HDFS failed",
+ sh.getRet() == 0);
+ }
+ sh.exec("hadoop fs -mkdir $INPUT");
+ assertTrue("Could not create $INPUT directory", sh.getRet() == 0);
+ // unpack resource
+ JarContent.unpackJarContainer(TestSqoopExport.class, '.' , null)
+ // upload data to HDFS
+ sh.exec("hadoop fs -put $DATA_DIR/sqoop-testtable.out input-dir/testtable/part-m-00000");
+ sh.exec("hadoop fs -put $DATA_DIR/sqoop-t_bool.out $INPUT/t_bool/part-m-00000");
+ sh.exec("hadoop fs -put $DATA_DIR/sqoop-t_date-export.out $INPUT/t_date/part-m-00000");
+ sh.exec("hadoop fs -put $DATA_DIR/sqoop-t_string.out $INPUT/t_string/part-m-00000");
+ sh.exec("hadoop fs -put $DATA_DIR/sqoop-t_fp.out $INPUT/t_fp/part-m-00000");
+ sh.exec("hadoop fs -put $DATA_DIR/sqoop-t_int.out $INPUT/t_int/part-m-00000");
+
+ //create db
+ sh.exec("cat $DATA_DIR/mysql-create-db.sql | $MYSQL_COMMAND");
+ //create tables
+ sh.exec("cat $DATA_DIR/mysql-create-tables.sql | $MYSQL_COMMAND");
+ }
+
+ @AfterClass
+ static void tearDown() {
+ if ('YES'.equals(System.getProperty('delete.testdata','no').toUpperCase())) {
+ sh.exec("hadoop fs -test -e $INPUT");
+ if (sh.getRet() == 0) {
+ // sh.exec("hadoop fs -rmr -skipTrash $INPUT");
+ assertTrue("Deletion of $INPUT from HDFS failed",
+ sh.getRet() == 0);
+ }
+ }
+ }
+
+ @Test
+ public void testBooleanExport() {
+ sh.exec("sqoop export $SQOOP_CONNECTION --table t_bool --export-dir $INPUT/t_bool");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("echo 'use mysqltestdb;select * from t_bool' | $MYSQL_COMMAND --skip-column-names | sed 's/\t/,/g' > t_bool.out");
+ assertEquals("sqoop export did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-t_bool-export.out t_bool.out").getRet());
+ }
+
+
+ @Test
+ public void testIntegerExport() {
+ sh.exec("sqoop export $SQOOP_CONNECTION --table t_int --export-dir $INPUT/t_int");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("echo 'use mysqltestdb;select * from t_int' | $MYSQL_COMMAND --skip-column-names | sed 's/\t/,/g' > t_int.out");
+ assertEquals("sqoop export did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-t_int.out t_int.out").getRet());
+ }
+
+ @Test
+ public void testFixedPointFloatingPointExport() {
+ sh.exec("sqoop export $SQOOP_CONNECTION --table t_fp --export-dir $INPUT/t_fp");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("echo 'use mysqltestdb;select * from t_fp' | $MYSQL_COMMAND --skip-column-names | sed 's/\t/,/g' > t_fp.out");
+ assertEquals("sqoop export did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-t_fp.out t_fp.out").getRet());
+ }
+
+ @Test
+ public void testDateTimeExport() {
+ sh.exec("sqoop export $SQOOP_CONNECTION --table t_date --export-dir $INPUT/t_date");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("echo 'use mysqltestdb;select * from t_date' | $MYSQL_COMMAND --skip-column-names | sed 's/\t/,/g' > t_date.out");
+ assertEquals("sqoop export did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-t_date.out t_date.out").getRet());
+ }
+
+ @Test
+ public void testStringExport() {
+ sh.exec("sqoop export $SQOOP_CONNECTION --table t_string --export-dir $INPUT/t_string");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("echo 'use mysqltestdb;select * from t_string' | $MYSQL_COMMAND --skip-column-names | sed 's/\t/,/g' > t_string.out");
+ assertEquals("sqoop export did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-t_string.out t_string.out").getRet());
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop/src/main/groovy/org/apache/bigtop/itest/sqoop/TestSqoopImport.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/src/main/groovy/org/apache/bigtop/itest/sqoop/TestSqoopImport.groovy b/bigtop-tests/test-artifacts/sqoop/src/main/groovy/org/apache/bigtop/itest/sqoop/TestSqoopImport.groovy
new file mode 100644
index 0000000..5845ee2
--- /dev/null
+++ b/bigtop-tests/test-artifacts/sqoop/src/main/groovy/org/apache/bigtop/itest/sqoop/TestSqoopImport.groovy
@@ -0,0 +1,264 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.itest.sqoop;
+
+import static org.junit.Assert.assertEquals
+import static org.junit.Assert.assertNotNull
+import static org.junit.Assert.assertTrue
+import org.junit.AfterClass
+import org.junit.BeforeClass
+import org.junit.Test
+
+import org.apache.bigtop.itest.JarContent
+import org.apache.bigtop.itest.shell.Shell
+
+class TestSqoopImport {
+ private static String mysql_user =
+ System.getenv("MYSQL_USER");
+ private static String mysql_password =
+ System.getenv("MYSQL_PASSWORD");
+ private static final String MYSQL_USER =
+ (mysql_user == null) ? "root" : mysql_user;
+ private static final String MYSQL_PASSWORD =
+ (mysql_password == null) ? "" : mysql_password;
+ private static final String MYSQL_HOST = System.getProperty("mysql.host", "localhost");
+ private static final String HADOOP_HOME =
+ System.getenv('HADOOP_HOME');
+ private static String streaming_home = System.getenv('STREAMING_HOME');
+ private static final String STREAMING_HOME =
+ (streaming_home == null) ? HADOOP_HOME + "/contrib/streaming" :
+ streaming_home;
+ private static final String SQOOP_HOME =
+ System.getenv("SQOOP_HOME");
+ static {
+ assertNotNull("HADOOP_HOME is not set", HADOOP_HOME);
+ assertNotNull("SQOOP_HOME is not set", SQOOP_HOME);
+ assertNotNull("mysql connector jar is required to be present in $SQOOP_HOME/lib",
+ JarContent.getJarName("$SQOOP_HOME/lib", "mysql-connector-java.*.jar"));
+ }
+ private static String sqoop_jar =
+ JarContent.getJarName(SQOOP_HOME, "sqoop-1.*.jar");
+ private static String streaming_jar =
+ JarContent.getJarName(STREAMING_HOME, "hadoop.*streaming.*.jar");
+ static {
+ assertNotNull("Can't find sqoop.jar", sqoop_jar);
+ assertNotNull("Can't find hadoop-streaming.jar", streaming_jar);
+ }
+ private static final String SQOOP_JAR = SQOOP_HOME + "/" + sqoop_jar;
+ private static final String STREAMING_JAR = STREAMING_HOME + "/" + streaming_jar;
+ private static final String MYSQL_COMMAND =
+ "mysql --user=$MYSQL_USER" +
+ (("".equals(MYSQL_PASSWORD)) ? "" : " --password=$MYSQL_PASSWORD");
+ private static final String MYSQL_DBNAME = System.getProperty("mysql.dbname", "mysqltestdb");
+ private static final String SQOOP_CONNECTION =
+ "--connect jdbc:mysql://$MYSQL_HOST/$MYSQL_DBNAME --username=$MYSQL_USER" +
+ (("".equals(MYSQL_PASSWORD)) ? "" : " --password=$MYSQL_PASSWORD");
+ static {
+ System.out.println("SQOOP_CONNECTION string is " + SQOOP_CONNECTION );
+ }
+ private static final String DATA_DIR = System.getProperty("data.dir", "mysql-files");
+ private static final String OUTPUT = System.getProperty("output.dir", "output-dir");
+ private static Shell sh = new Shell("/bin/bash -s");
+
+ @BeforeClass
+ static void setUp() {
+ sh.exec("hadoop fs -test -e $OUTPUT");
+ if (sh.getRet() == 0) {
+ sh.exec("hadoop fs -rmr -skipTrash $OUTPUT");
+ assertTrue("Deletion of previous $OUTPUT from HDFS failed",
+ sh.getRet() == 0);
+ }
+ sh.exec("hadoop fs -mkdir $OUTPUT");
+ assertTrue("Could not create $OUTPUT directory", sh.getRet() == 0);
+ // unpack resource
+ JarContent.unpackJarContainer(TestSqoopImport.class, '.' , null)
+ // create the database
+ sh.exec("cat $DATA_DIR/mysql-create-db.sql | $MYSQL_COMMAND");
+ //create tables
+ sh.exec("cat $DATA_DIR/mysql-create-tables.sql | $MYSQL_COMMAND");
+ //populate data
+ sh.exec("cat $DATA_DIR/mysql-insert-data.sql | $MYSQL_COMMAND");
+ }
+
+ @AfterClass
+ static void tearDown() {
+ if ('YES'.equals(System.getProperty('delete.testdata','no').toUpperCase())) {
+ sh.exec("hadoop fs -test -e $OUTPUT");
+ if (sh.getRet() == 0) {
+ sh.exec("hadoop fs -rmr -skipTrash $OUTPUT");
+ assertTrue("Deletion of $OUTPUT from HDFS failed",
+ sh.getRet() == 0);
+ }
+ }
+ }
+
+ @Test
+ public void testBooleanImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table t_bool --target-dir $OUTPUT/t_bool");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/t_bool/part-* > t_bool.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-t_bool.out t_bool.out").getRet());
+ }
+
+
+ @Test
+ public void testIntegerImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table t_int --target-dir $OUTPUT/t_int");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/t_int/part-* > t_int.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-t_int.out t_int.out").getRet());
+ }
+
+ @Test
+ public void testFixedPointFloatingPointImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table t_fp --target-dir $OUTPUT/t_fp");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/t_fp/part-* > t_fp.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-t_fp.out t_fp.out").getRet());
+ }
+
+ @Test
+ public void testDateTimeImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table t_date --target-dir $OUTPUT/t_date");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/t_date/part-* > t_date.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-t_date.out t_date.out").getRet());
+ }
+
+ @Test
+ public void testStringImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table t_string --target-dir $OUTPUT/t_string");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/t_string/part-* > t_string.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-t_string.out t_string.out").getRet());
+ }
+
+ @Test
+ public void testAppendImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --target-dir $OUTPUT/append");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ //import again with append
+ sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --append --target-dir $OUTPUT/append");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/append/part-* > append.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-append.out append.out").getRet());
+ }
+
+ @Test
+ public void testColumnsImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --columns id,fname --target-dir $OUTPUT/columns");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/columns/part-* > columns.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-columns.out columns.out").getRet());
+ }
+
+ @Test
+ public void testDirectImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --direct --target-dir $OUTPUT/direct");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/direct/part-* > direct.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-testtable.out direct.out").getRet());
+ }
+
+ @Test
+ public void testNumMappersImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --num-mappers 1 --target-dir $OUTPUT/num-mappers");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/num-mappers/part-*0 > num-mappers.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-testtable.out num-mappers.out").getRet());
+ }
+
+ @Test
+ public void testQueryImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --query 'select t1.id as id, t2.fname as fname from testtable as t1 join testtable2 as t2 on (t1.id = t2.id ) where t1.id < 3 AND \$CONDITIONS' --split-by t1.id --target-dir $OUTPUT/query");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/query/part-* > query.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-query.out query.out").getRet());
+ }
+
+ @Test
+ public void testSplityByImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --split-by fname --target-dir $OUTPUT/split-by");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/split-by/part-* > split-by.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-testtable.out split-by.out").getRet());
+ }
+
+ @Test
+ public void testWarehouseDirImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --warehouse-dir $OUTPUT/warehouse-dir");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/warehouse-dir/testtable/part-* > warehouse-dir.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-testtable.out warehouse-dir.out").getRet());
+ }
+
+ @Test
+ public void testWhereClauseImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --where \"id < 5\" --target-dir $OUTPUT/where-clause");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/where-clause/part-* > where-clause.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-where-clause.out where-clause.out").getRet());
+ }
+
+ @Test
+ public void testNullStringImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table testnullvalues --null-string mynullstring --target-dir $OUTPUT/null-string");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/null-string/part-* > null-string.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-null-string.out null-string.out").getRet());
+ }
+
+ @Test
+ public void testNullNonStringImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table testnullvalues --null-non-string 10 --target-dir $OUTPUT/non-null-string");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/non-null-string/part-* > non-null-string.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-null-non-string.out non-null-string.out").getRet());
+ }
+
+ //database name is hardcoded here
+ @Test
+ public void testImportAllTables() {
+ String SQOOP_CONNECTION_IMPORT_ALL =
+ "--connect jdbc:mysql://$MYSQL_HOST/mysqltestdb2 --username=$MYSQL_USER" +
+ (("".equals(MYSQL_PASSWORD)) ? "" : " --password=$MYSQL_PASSWORD");
+
+ sh.exec("sqoop import-all-tables $SQOOP_CONNECTION_IMPORT_ALL --warehouse-dir $OUTPUT/alltables");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/alltables/testtable*/part-* > all-tables.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-all-tables.out all-tables.out").getRet());
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/mysql-create-db.sql
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/mysql-create-db.sql b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/mysql-create-db.sql
new file mode 100644
index 0000000..7746241
--- /dev/null
+++ b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/mysql-create-db.sql
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+drop database if exists mysqltestdb;
+
+create database mysqltestdb;
+
+use mysqltestdb;
+
+grant all privileges on mysqltestdb.* to 'root'@'%';
+
+flush privileges;
+
+/* create database2 for import-all */
+
+drop database if exists mysqltestdb2;
+
+create database mysqltestdb2;
+
+use mysqltestdb2;
+
+grant all privileges on mysqltestdb2.* to 'root'@'%';
+
+flush privileges;
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/mysql-create-tables.sql
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/mysql-create-tables.sql b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/mysql-create-tables.sql
new file mode 100644
index 0000000..5a4a7ce
--- /dev/null
+++ b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/mysql-create-tables.sql
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* functional test db tables */
+use mysqltestdb;
+
+drop table if exists t_bool;
+create table t_bool ( pri int not null auto_increment, a boolean, primary key (pri));
+
+drop table if exists t_int;
+create table t_int ( pri int not null auto_increment,a tinyint, b smallint, c mediumint , d int, e integer , f bigint,primary key (pri));
+
+drop table if exists t_fp;
+create table t_fp (pri int not null auto_increment, a decimal (6,4), b float (6,4), c double precision (6,4),d float (4),primary key (pri));
+
+drop table if exists t_date;
+create table t_date (pri int not null auto_increment,a date , c timestamp, d time , e year(4),primary key (pri));
+
+drop table if exists t_string;
+create table t_string (pri int not null auto_increment, a char (5), b varchar (5), c text , d tinytext, e mediumtext, f mediumtext, g longtext, h enum ('A','B'),primary key (pri));
+
+drop table if exists testtable;
+create table testtable ( id int , fname varchar (20), lname varchar(20) , primary key (id));
+
+drop table if exists testtable2;
+create table testtable2 ( id int , fname varchar (20), lname varchar(20) , primary key (id));
+
+drop table if exists testnullvalues;
+create table testnullvalues ( id int , a int , b varchar(20) , primary key (id));
+
+/* import-all test db-tables */
+
+use mysqltestdb2;
+
+drop table if exists testtable;
+create table testtable ( id int , fname varchar (20), lname varchar(20) , primary key (id));
+
+drop table if exists testtable2;
+create table testtable2 ( id int , fname varchar (20), lname varchar(20) , primary key (id));
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/mysql-insert-data.sql
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/mysql-insert-data.sql b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/mysql-insert-data.sql
new file mode 100644
index 0000000..c8aef22
--- /dev/null
+++ b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/mysql-insert-data.sql
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use mysqltestdb;
+
+insert into t_int (a,b,c,d,e,f) values (127,32767,8388607,2147483647,-2147483648,9223372036854775807);
+
+insert into t_bool (a) values (1);
+
+insert into t_fp(a,b,c,d) values (-99.9999,-99.9999,99.9999,99.99);
+
+insert into t_date (a,c,d,e) values ('9999-12-31',19731230153000,'23:59:59','2155');
+
+insert into t_string (a,b,c,d,e,f,g,h) values ('aaa','aaa','0GIQM3CONg5jAUloRoP7f76TJrYkYnto5i76IUDbKuG1dMSP5znIM6Ct3qc1WdRCD9THFjWXBIxOoAqhOgdFQEwICX7fYN4n9P3AOoo3ZrnUBu3rtuwf9Y2HXJvUBgAqs44Ypdg6iv511JQSufMeKvbLIpIbk9saJW82EeTgoWkAS7lhI0HJXmvyDkJEqOc0VxdT0ySR','0GIQM3CONg5jAUloRoP7f76TJrYkYnto5i76IUDbKuG1dMSP5znIM6Ct3qc1WdRCD9THFjWXBIxOoAqhOgdFQEwICX7fYN4n9P3AOoo3ZrnUBu3rtuwf9Y2HXJvUBgAqs44Ypdg6iv511JQSufMeKvbLIpIbk9saJW82EeTgoWkAS7lhI0HJXmvyDkJEqOc0VxdT0ySR','0GIQM3CONg5jAUloRoP7f76TJrYkYnto5i76IUDbKuG1dMSP5znIM6Ct3qc1WdRCD9THFjWXBIxOoAqhOgdFQEwICX7fYN4n9P3AOoo3ZrnUBu3rtuwf9Y2HXJvUBgAqs44Ypdg6iv511JQSufMeKvbLIpIbk9saJW82EeTgoWkAS7lhI0HJXmvyDkJEqOc0VxdT0ySR','0GIQM3CONg5jAUloRoP7f76TJrYkYnto5i76IUDbKuG1dMSP5znIM6Ct3qc1WdRCD9THFjWXBIxOoAqhOgdFQEwICX7fYN4n9P3AOoo3ZrnUBu3rtuwf9Y2HXJvUBgAqs44Ypdg6iv511JQSufMeKvbLIpIbk9saJW82EeTgoWkAS7lhI0HJXmvyDkJEqOc0VxdT0ySR','0GIQM3CONg5jAUloRoP7f76TJrYkYnto5i76IUDbKuG1dMSP5znIM6Ct3qc1WdRCD9THFjWXBIxOoAqhOgdFQEwICX7fYN4n9P3AOoo3ZrnUBu3rtuwf9Y2HXJvUB
gAqs44Ypdg6iv511JQSufMeKvbLIpIbk9saJW82EeTgoWkAS7lhI0HJXmvyDkJEqOc0VxdT0ySR','A');
+
+insert into testtable values (1,'aaa','aaa');
+insert into testtable values (2,'bbb','bbb');
+insert into testtable values (3,'ccc','ccc');
+insert into testtable values (4,'ddd','ddd');
+insert into testtable values (5,'eee','eee');
+insert into testtable values (6,'fff','fff');
+insert into testtable values (7,'ggg','ggg');
+insert into testtable values (8,'hhh','hhh');
+insert into testtable values (9,'iii','iii');
+insert into testtable values (10,'jjj','jjj');
+
+
+insert into testtable2 values (1,'111','111');
+insert into testtable2 values (2,'222','222');
+insert into testtable2 values (3,'333','333');
+
+
+insert into testnullvalues (id) values (1);
+insert into testnullvalues (id,a) values (2,2);
+insert into testnullvalues (id,b) values (3,'aaa');
+
+/* data for import-all test */
+
+use mysqltestdb2;
+
+insert into testtable values (1,'aaa','aaa');
+insert into testtable2 values (1,'aaa','aaa');
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-all-tables.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-all-tables.out b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-all-tables.out
new file mode 100644
index 0000000..855e550
--- /dev/null
+++ b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-all-tables.out
@@ -0,0 +1,2 @@
+1,aaa,aaa
+1,aaa,aaa
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-append.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-append.out b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-append.out
new file mode 100644
index 0000000..98d09c4
--- /dev/null
+++ b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-append.out
@@ -0,0 +1,20 @@
+1,aaa,aaa
+2,bbb,bbb
+3,ccc,ccc
+4,ddd,ddd
+5,eee,eee
+6,fff,fff
+7,ggg,ggg
+8,hhh,hhh
+9,iii,iii
+10,jjj,jjj
+1,aaa,aaa
+2,bbb,bbb
+3,ccc,ccc
+4,ddd,ddd
+5,eee,eee
+6,fff,fff
+7,ggg,ggg
+8,hhh,hhh
+9,iii,iii
+10,jjj,jjj
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-columns.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-columns.out b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-columns.out
new file mode 100644
index 0000000..dd3c973
--- /dev/null
+++ b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-columns.out
@@ -0,0 +1,10 @@
+1,aaa
+2,bbb
+3,ccc
+4,ddd
+5,eee
+6,fff
+7,ggg
+8,hhh
+9,iii
+10,jjj
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-null-non-string.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-null-non-string.out b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-null-non-string.out
new file mode 100644
index 0000000..c1ef144
--- /dev/null
+++ b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-null-non-string.out
@@ -0,0 +1,3 @@
+1,10,null
+2,2,null
+3,10,aaa
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-null-string.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-null-string.out b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-null-string.out
new file mode 100644
index 0000000..acddc7b
--- /dev/null
+++ b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-null-string.out
@@ -0,0 +1,3 @@
+1,null,mynullstring
+2,2,mynullstring
+3,null,aaa
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-query.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-query.out b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-query.out
new file mode 100644
index 0000000..786a05c
--- /dev/null
+++ b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-query.out
@@ -0,0 +1,2 @@
+1,111
+2,222
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-t_bool-export.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-t_bool-export.out b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-t_bool-export.out
new file mode 100644
index 0000000..9474240
--- /dev/null
+++ b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-t_bool-export.out
@@ -0,0 +1 @@
+1,1
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-t_bool.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-t_bool.out b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-t_bool.out
new file mode 100644
index 0000000..5f49b20
--- /dev/null
+++ b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-t_bool.out
@@ -0,0 +1 @@
+1,true
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a6ca5acd/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-t_date-export.out
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-t_date-export.out b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-t_date-export.out
new file mode 100644
index 0000000..992838e
--- /dev/null
+++ b/bigtop-tests/test-artifacts/sqoop/src/main/resources/mysql-files/sqoop-t_date-export.out
@@ -0,0 +1 @@
+1,"9999-12-31","9999-12-31 23:59:59","1973-12-30 15:30:00","23:59:59","2155"