You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by an...@apache.org on 2017/05/15 08:10:09 UTC
[1/4] sqoop git commit: SQOOP-3174: Add SQLServer manual tests to 3rd
party test suite
Repository: sqoop
Updated Branches:
refs/heads/trunk 0ca73d4e7 -> 558bdaea9
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerSplitByManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerSplitByManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerSplitByManualTest.java
deleted file mode 100644
index 67e2cae..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerSplitByManualTest.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sqoop.manager.sqlserver;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
-import com.cloudera.sqoop.orm.CompilationManager;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
-import com.cloudera.sqoop.testutil.SeqFileReader;
-import com.cloudera.sqoop.tool.ImportTool;
-import com.cloudera.sqoop.util.ClassLoaderStack;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * Test that --split-by works in SQL Server.
- *
- * This uses JDBC to import data from an SQLServer database to HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerSplitByManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerSplitByManualTest extends ImportJobTestCase {
-
- @Before
- public void setUp() {
- super.setUp();
- MSSQLTestUtils utils = new MSSQLTestUtils();
- try {
- utils.createTableFromSQL(MSSQLTestUtils.CREATE_TALBE_LINEITEM);
- utils.populateLineItem();
- } catch (SQLException e) {
- LOG.error("Setup fail with SQLException: " + StringUtils.stringifyException(e));
- fail("Setup fail with SQLException: " + e.toString());
- }
-
- }
-
- @After
- public void tearDown() {
- super.tearDown();
- MSSQLTestUtils utils = new MSSQLTestUtils();
- try {
- utils.dropTableIfExists("TPCH1M_LINEITEM");
- } catch (SQLException e) {
- LOG.error("TearDown fail with SQLException: " + StringUtils.stringifyException(e));
- fail("TearDown fail with SQLException: " + e.toString());
- }
- }
-
- /**
- * Create the argv to pass to Sqoop.
- *
- * @return the argv as an array of strings.
- */
- protected String[] getArgv(boolean includeHadoopFlags, String[] colNames,
- String splitByCol) {
- String columnsString = "";
- for (String col : colNames) {
- columnsString += col + ",";
- }
-
- ArrayList<String> args = new ArrayList<String>();
-
- if (includeHadoopFlags) {
- CommonArgs.addHadoopFlags(args);
- }
-
- args.add("--table");
- args.add("tpch1m_lineitem");
- args.add("--columns");
- args.add(columnsString);
- args.add("--split-by");
- args.add("L_ORDERKEY");
- args.add("--warehouse-dir");
- args.add(getWarehouseDir());
- args.add("--connect");
- args.add(getConnectString());
- args.add("--as-sequencefile");
- args.add("--num-mappers");
- args.add("1");
-
- return args.toArray(new String[0]);
- }
-
- /**
- * Given a comma-delimited list of integers, grab and parse the first int.
- *
- * @param str
- * a comma-delimited list of values, the first of which is an
- * int.
- * @return the first field in the string, cast to int
- */
- private int getFirstInt(String str) {
- String[] parts = str.split(",");
- return Integer.parseInt(parts[0]);
- }
-
- public void runSplitByTest(String splitByCol, int expectedSum)
- throws IOException {
-
- String[] columns = new String[] { "L_ORDERKEY", "L_PARTKEY",
- "L_SUPPKEY", "L_LINENUMBER", "L_QUANTITY", "L_EXTENDEDPRICE",
- "L_DISCOUNT", "L_TAX", "L_RETURNFLAG", "L_LINESTATUS",
- "L_SHIPDATE", "L_COMMITDATE", "L_RECEIPTDATE",
- "L_SHIPINSTRUCT", "L_SHIPMODE", "L_COMMENT", };
- ClassLoader prevClassLoader = null;
- SequenceFile.Reader reader = null;
-
- String[] argv = getArgv(true, columns, splitByCol);
- runImport(argv);
- try {
- SqoopOptions opts = new ImportTool().parseArguments(getArgv(false,
- columns, splitByCol), null, null, true);
-
- CompilationManager compileMgr = new CompilationManager(opts);
- String jarFileName = compileMgr.getJarFilename();
- LOG.debug("Got jar from import job: " + jarFileName);
-
- prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
- getTableName());
-
- reader = SeqFileReader.getSeqFileReader(getDataFilePath()
- .toString());
-
- // here we can actually instantiate (k, v) pairs.
- Configuration conf = new Configuration();
- Object key = ReflectionUtils
- .newInstance(reader.getKeyClass(), conf);
- Object val = ReflectionUtils.newInstance(reader.getValueClass(),
- conf);
-
- // We know that these values are two ints separated by a ','
- // character.
- // Since this is all dynamic, though, we don't want to actually link
- // against the class and use its methods. So we just parse this back
- // into int fields manually. Sum them up and ensure that we get the
- // expected total for the first column, to verify that we got all
- // the
- // results from the db into the file.
-
- // Sum up everything in the file.
- int curSum = 0;
- while (reader.next(key) != null) {
- reader.getCurrentValue(val);
- curSum += getFirstInt(val.toString());
- }
- System.out.println("Sum : e,c" + expectedSum + " : " + curSum);
- assertEquals("Total sum of first db column mismatch", expectedSum,
- curSum);
- } catch (InvalidOptionsException ioe) {
- LOG.error(StringUtils.stringifyException(ioe));
- fail(ioe.toString());
- } catch (ParseException pe) {
- LOG.error(StringUtils.stringifyException(pe));
- fail(pe.toString());
- } finally {
- IOUtils.closeStream(reader);
-
- if (null != prevClassLoader) {
- ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
- }
- }
- }
-
- @Test
- public void testSplitByFirstCol() throws IOException {
- String splitByCol = "L_ORDERKEY";
- runSplitByTest(splitByCol, 10);
- }
-
- @Test
- public void testSplitBySecondCol() throws IOException {
- String splitByCol = "L_PARTKEY";
- runSplitByTest(splitByCol, 10);
- }
-
- protected boolean useHsqldbTestServer() {
-
- return false;
- }
-
- protected String getConnectString() {
- return MSSQLTestUtils.getDBConnectString();
- }
-
- protected String getTableName() {
- return "tpch1m_lineitem";
- }
-
- /**
- * Drop a table if it already exists in the database.
- *
- * @param table
- * the name of the table to drop.
- * @throws SQLException
- * if something goes wrong.
- */
- protected void dropTableIfExists(String table) throws SQLException {
- Connection conn = getManager().getConnection();
- String sqlStmt = "IF OBJECT_ID('" + table
- + "') IS NOT NULL DROP TABLE " + table;
- System.out.println("@abhi SQL for drop :" + sqlStmt);
- PreparedStatement statement = conn.prepareStatement(sqlStmt,
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- conn.commit();
- } finally {
- statement.close();
- }
- }
-
- protected SqoopOptions getSqoopOptions(Configuration conf) {
- SqoopOptions opts = new SqoopOptions(conf);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- opts.setUsername(username);
- opts.setPassword(password);
- return opts;
-
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerSplitByTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerSplitByTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerSplitByTest.java
new file mode 100644
index 0000000..4894b21
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerSplitByTest.java
@@ -0,0 +1,271 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.sqlserver;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
+import com.cloudera.sqoop.orm.CompilationManager;
+import com.cloudera.sqoop.testutil.CommonArgs;
+import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import com.cloudera.sqoop.testutil.SeqFileReader;
+import com.cloudera.sqoop.tool.ImportTool;
+import com.cloudera.sqoop.util.ClassLoaderStack;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Test that --split-by works in SQL Server.
+ *
+ * This uses JDBC to import data from an SQLServer database to HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerSplitByTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerSplitByTest extends ImportJobTestCase {
+
+ @Before
+ public void setUp() {
+ super.setUp();
+ MSSQLTestUtils utils = new MSSQLTestUtils();
+ try {
+ utils.createTableFromSQL(MSSQLTestUtils.CREATE_TALBE_LINEITEM);
+ utils.populateLineItem();
+ } catch (SQLException e) {
+ LOG.error("Setup fail with SQLException: " + StringUtils.stringifyException(e));
+ fail("Setup fail with SQLException: " + e.toString());
+ }
+
+ }
+
+ @After
+ public void tearDown() {
+ super.tearDown();
+ MSSQLTestUtils utils = new MSSQLTestUtils();
+ try {
+ utils.dropTableIfExists("TPCH1M_LINEITEM");
+ } catch (SQLException e) {
+ LOG.error("TearDown fail with SQLException: " + StringUtils.stringifyException(e));
+ fail("TearDown fail with SQLException: " + e.toString());
+ }
+ }
+
+ /**
+ * Create the argv to pass to Sqoop.
+ *
+ * @return the argv as an array of strings.
+ */
+ protected String[] getArgv(boolean includeHadoopFlags, String[] colNames,
+ String splitByCol) {
+ String columnsString = "";
+ for (String col : colNames) {
+ columnsString += col + ",";
+ }
+
+ ArrayList<String> args = new ArrayList<String>();
+
+ if (includeHadoopFlags) {
+ CommonArgs.addHadoopFlags(args);
+ }
+
+ args.add("--table");
+ args.add("tpch1m_lineitem");
+ args.add("--columns");
+ args.add(columnsString);
+ args.add("--split-by");
+ args.add("L_ORDERKEY");
+ args.add("--warehouse-dir");
+ args.add(getWarehouseDir());
+ args.add("--connect");
+ args.add(getConnectString());
+ args.add("--as-sequencefile");
+ args.add("--num-mappers");
+ args.add("1");
+
+ return args.toArray(new String[0]);
+ }
+
+ /**
+ * Given a comma-delimited list of integers, grab and parse the first int.
+ *
+ * @param str
+ * a comma-delimited list of values, the first of which is an
+ * int.
+ * @return the first field in the string, cast to int
+ */
+ private int getFirstInt(String str) {
+ String[] parts = str.split(",");
+ return Integer.parseInt(parts[0]);
+ }
+
+ public void runSplitByTest(String splitByCol, int expectedSum)
+ throws IOException {
+
+ String[] columns = new String[] { "L_ORDERKEY", "L_PARTKEY",
+ "L_SUPPKEY", "L_LINENUMBER", "L_QUANTITY", "L_EXTENDEDPRICE",
+ "L_DISCOUNT", "L_TAX", "L_RETURNFLAG", "L_LINESTATUS",
+ "L_SHIPDATE", "L_COMMITDATE", "L_RECEIPTDATE",
+ "L_SHIPINSTRUCT", "L_SHIPMODE", "L_COMMENT", };
+ ClassLoader prevClassLoader = null;
+ SequenceFile.Reader reader = null;
+
+ String[] argv = getArgv(true, columns, splitByCol);
+ runImport(argv);
+ try {
+ SqoopOptions opts = new ImportTool().parseArguments(getArgv(false,
+ columns, splitByCol), null, null, true);
+
+ CompilationManager compileMgr = new CompilationManager(opts);
+ String jarFileName = compileMgr.getJarFilename();
+ LOG.debug("Got jar from import job: " + jarFileName);
+
+ prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
+ getTableName());
+
+ reader = SeqFileReader.getSeqFileReader(getDataFilePath()
+ .toString());
+
+ // here we can actually instantiate (k, v) pairs.
+ Configuration conf = new Configuration();
+ Object key = ReflectionUtils
+ .newInstance(reader.getKeyClass(), conf);
+ Object val = ReflectionUtils.newInstance(reader.getValueClass(),
+ conf);
+
+ // We know that these values are two ints separated by a ','
+ // character.
+ // Since this is all dynamic, though, we don't want to actually link
+ // against the class and use its methods. So we just parse this back
+ // into int fields manually. Sum them up and ensure that we get the
+ // expected total for the first column, to verify that we got all
+ // the
+ // results from the db into the file.
+
+ // Sum up everything in the file.
+ int curSum = 0;
+ while (reader.next(key) != null) {
+ reader.getCurrentValue(val);
+ curSum += getFirstInt(val.toString());
+ }
+ System.out.println("Sum : e,c" + expectedSum + " : " + curSum);
+ assertEquals("Total sum of first db column mismatch", expectedSum,
+ curSum);
+ } catch (InvalidOptionsException ioe) {
+ LOG.error(StringUtils.stringifyException(ioe));
+ fail(ioe.toString());
+ } catch (ParseException pe) {
+ LOG.error(StringUtils.stringifyException(pe));
+ fail(pe.toString());
+ } finally {
+ IOUtils.closeStream(reader);
+
+ if (null != prevClassLoader) {
+ ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
+ }
+ }
+ }
+
+ @Test
+ public void testSplitByFirstCol() throws IOException {
+ String splitByCol = "L_ORDERKEY";
+ runSplitByTest(splitByCol, 10);
+ }
+
+ @Test
+ public void testSplitBySecondCol() throws IOException {
+ String splitByCol = "L_PARTKEY";
+ runSplitByTest(splitByCol, 10);
+ }
+
+ protected boolean useHsqldbTestServer() {
+
+ return false;
+ }
+
+ protected String getConnectString() {
+ return MSSQLTestUtils.getDBConnectString();
+ }
+
+ protected String getTableName() {
+ return "tpch1m_lineitem";
+ }
+
+ /**
+ * Drop a table if it already exists in the database.
+ *
+ * @param table
+ * the name of the table to drop.
+ * @throws SQLException
+ * if something goes wrong.
+ */
+ protected void dropTableIfExists(String table) throws SQLException {
+ Connection conn = getManager().getConnection();
+ String sqlStmt = "IF OBJECT_ID('" + table
+ + "') IS NOT NULL DROP TABLE " + table;
+ System.out.println("@abhi SQL for drop :" + sqlStmt);
+ PreparedStatement statement = conn.prepareStatement(sqlStmt,
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ statement.executeUpdate();
+ conn.commit();
+ } finally {
+ statement.close();
+ }
+ }
+
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ SqoopOptions opts = new SqoopOptions(conf);
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ opts.setUsername(username);
+ opts.setPassword(password);
+ return opts;
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerWhereManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerWhereManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerWhereManualTest.java
deleted file mode 100644
index 700fbba..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerWhereManualTest.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sqoop.manager.sqlserver;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
-import com.cloudera.sqoop.orm.CompilationManager;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
-import com.cloudera.sqoop.testutil.SeqFileReader;
-import com.cloudera.sqoop.tool.ImportTool;
-import com.cloudera.sqoop.util.ClassLoaderStack;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * Test that --where works in SQL Server.
- *
- * This uses JDBC to import data from an SQLServer database to HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerWhereManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerWhereManualTest extends ImportJobTestCase {
-
- @Before
- public void setUp(){
- super.setUp();
- MSSQLTestUtils utils = new MSSQLTestUtils();
- try {
- utils.createTableFromSQL(MSSQLTestUtils.CREATE_TALBE_LINEITEM);
- utils.populateLineItem();
- } catch (SQLException e) {
- LOG.error("Setup fail with SQLException: " + StringUtils.stringifyException(e));
- fail("Setup fail with SQLException: " + e.toString());
- }
-
- }
-
- @After
- public void tearDown(){
- super.tearDown();
- MSSQLTestUtils utils = new MSSQLTestUtils();
- try {
- utils.dropTableIfExists("TPCH1M_LINEITEM");
- } catch (SQLException e) {
- LOG.error("TearDown fail with SQLException: " + StringUtils.stringifyException(e));
- fail("TearDown fail with SQLException: " + e.toString());
- }
- }
- /**
- * Create the argv to pass to Sqoop.
- *
- * @return the argv as an array of strings.
- */
- protected String[] getArgv(boolean includeHadoopFlags, String[] colNames,
- String whereClause) {
- String columnsString = "";
- for (String col : colNames) {
- columnsString += col + ",";
- }
-
- ArrayList<String> args = new ArrayList<String>();
-
- if (includeHadoopFlags) {
- CommonArgs.addHadoopFlags(args);
- }
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- args.add("--table");
- args.add(getTableName());
- args.add("--columns");
- args.add(columnsString);
- args.add("--where");
- args.add(whereClause);
- args.add("--split-by");
- args.add("L_ORDERKEY");
- args.add("--warehouse-dir");
- args.add(getWarehouseDir());
- args.add("--connect");
- args.add(getConnectString());
- args.add("--username");
- args.add(username);
- args.add("--password");
- args.add(password);
- args.add("--as-sequencefile");
- args.add("--num-mappers");
- args.add("1");
-
- return args.toArray(new String[0]);
- }
-
- /**
- * Given a comma-delimited list of integers, grab and parse the first int.
- *
- * @param str
- * a comma-delimited list of values, the first of which is an
- * int.
- * @return the first field in the string, cast to int
- */
- private int getFirstInt(String str) {
- String[] parts = str.split(",");
- return Integer.parseInt(parts[0]);
- }
-
- public void runWhereTest(String whereClause, String firstValStr,
- int numExpectedResults, int expectedSum) throws IOException {
-
- String[] columns = MSSQLTestUtils.getColumns();
- ClassLoader prevClassLoader = null;
- SequenceFile.Reader reader = null;
-
- String[] argv = getArgv(true, columns, whereClause);
- runImport(argv);
- try {
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
-
- SqoopOptions opts = new ImportTool().parseArguments(getArgv(false,
- columns, whereClause), null, null, true);
- opts.setUsername(username);
- opts.setPassword(password);
- CompilationManager compileMgr = new CompilationManager(opts);
- String jarFileName = compileMgr.getJarFilename();
-
- prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
- getTableName());
-
- reader = SeqFileReader.getSeqFileReader(getDataFilePath()
- .toString());
-
- // here we can actually instantiate (k, v) pairs.
- Configuration conf = new Configuration();
- Object key = ReflectionUtils
- .newInstance(reader.getKeyClass(), conf);
- Object val = ReflectionUtils.newInstance(reader.getValueClass(),
- conf);
-
- if (reader.next(key) == null) {
- fail("Empty SequenceFile during import");
- }
-
- // make sure that the value we think should be at the top, is.
- reader.getCurrentValue(val);
- assertEquals("Invalid ordering within sorted SeqFile", firstValStr,
- val.toString());
-
- // We know that these values are two ints separated by a ','
- // character.
- // Since this is all dynamic, though, we don't want to actually link
- // against the class and use its methods. So we just parse this back
- // into int fields manually. Sum them up and ensure that we get the
- // expected total for the first column, to verify that we got all
- // the
- // results from the db into the file.
- int curSum = getFirstInt(val.toString());
- int totalResults = 1;
-
- // now sum up everything else in the file.
- while (reader.next(key) != null) {
- reader.getCurrentValue(val);
- curSum += getFirstInt(val.toString());
- totalResults++;
- }
-
- assertEquals("Total sum of first db column mismatch", expectedSum,
- curSum);
- assertEquals("Incorrect number of results for query",
- numExpectedResults, totalResults);
- } catch (InvalidOptionsException ioe) {
- fail(ioe.toString());
- } catch (ParseException pe) {
- fail(pe.toString());
- } finally {
- IOUtils.closeStream(reader);
-
- if (null != prevClassLoader) {
- ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
- }
- }
- }
-
- @Test
- public void testSingleClauseWhere() throws IOException {
- String whereClause = "L_ORDERKEY > 0 ";
- runWhereTest(whereClause,
- "1,2,3,4,5,6.00,7.00,8.00,AB,CD,abcd,efgh,hijk,dothis,likethis,nocomments"
- + "\n", 4, 10);
- }
-
- @Test
- public void testMultiClauseWhere() throws IOException {
- String whereClause = "L_ORDERKEY > 1 AND L_PARTKEY < 4";
- runWhereTest(whereClause,
- "2,3,4,5,6,7.00,8.00,9.00,AB,CD,abcd,efgh,hijk,dothis,likethis,nocomments"
- + "\n", 1, 2);
- }
-
- protected boolean useHsqldbTestServer() {
-
- return false;
- }
-
- protected String getConnectString() {
- return MSSQLTestUtils.getDBConnectString();
- }
-
- /**
- * Drop a table if it already exists in the database.
- *
- * @param table
- * the name of the table to drop.
- * @throws SQLException
- * if something goes wrong.
- */
- protected void dropTableIfExists(String table) throws SQLException {
- Connection conn = getManager().getConnection();
- String sqlStmt = "IF OBJECT_ID('" + table
- + "') IS NOT NULL DROP TABLE " + table;
-
- PreparedStatement statement = conn.prepareStatement(sqlStmt,
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- conn.commit();
- } finally {
- statement.close();
- }
- }
-
- protected SqoopOptions getSqoopOptions(Configuration conf) {
- SqoopOptions opt = new SqoopOptions(conf);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- SqoopOptions opts = new SqoopOptions(conf);
- opts.setUsername(username);
- opts.setPassword(password);
-
- return opt;
- }
-
- protected String getTableName() {
- return "tpch1m_lineitem";
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerWhereTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerWhereTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerWhereTest.java
new file mode 100644
index 0000000..2c5dbd3
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerWhereTest.java
@@ -0,0 +1,292 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.sqlserver;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
+import com.cloudera.sqoop.orm.CompilationManager;
+import com.cloudera.sqoop.testutil.CommonArgs;
+import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import com.cloudera.sqoop.testutil.SeqFileReader;
+import com.cloudera.sqoop.tool.ImportTool;
+import com.cloudera.sqoop.util.ClassLoaderStack;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Test that --where works in SQL Server.
+ *
+ * This uses JDBC to import data from an SQLServer database to HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerWhereTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerWhereTest extends ImportJobTestCase {
+
+ @Before
+ public void setUp(){
+ super.setUp();
+ MSSQLTestUtils utils = new MSSQLTestUtils();
+ try {
+ utils.createTableFromSQL(MSSQLTestUtils.CREATE_TALBE_LINEITEM);
+ utils.populateLineItem();
+ } catch (SQLException e) {
+ LOG.error("Setup fail with SQLException: " + StringUtils.stringifyException(e));
+ fail("Setup fail with SQLException: " + e.toString());
+ }
+
+ }
+
+ @After
+ public void tearDown(){
+ super.tearDown();
+ MSSQLTestUtils utils = new MSSQLTestUtils();
+ try {
+ utils.dropTableIfExists("TPCH1M_LINEITEM");
+ } catch (SQLException e) {
+ LOG.error("TearDown fail with SQLException: " + StringUtils.stringifyException(e));
+ fail("TearDown fail with SQLException: " + e.toString());
+ }
+ }
+ /**
+ * Create the argv to pass to Sqoop.
+ *
+ * @return the argv as an array of strings.
+ */
+ protected String[] getArgv(boolean includeHadoopFlags, String[] colNames,
+ String whereClause) {
+ String columnsString = "";
+ for (String col : colNames) {
+ columnsString += col + ",";
+ }
+
+ ArrayList<String> args = new ArrayList<String>();
+
+ if (includeHadoopFlags) {
+ CommonArgs.addHadoopFlags(args);
+ }
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ args.add("--table");
+ args.add(getTableName());
+ args.add("--columns");
+ args.add(columnsString);
+ args.add("--where");
+ args.add(whereClause);
+ args.add("--split-by");
+ args.add("L_ORDERKEY");
+ args.add("--warehouse-dir");
+ args.add(getWarehouseDir());
+ args.add("--connect");
+ args.add(getConnectString());
+ args.add("--username");
+ args.add(username);
+ args.add("--password");
+ args.add(password);
+ args.add("--as-sequencefile");
+ args.add("--num-mappers");
+ args.add("1");
+
+ return args.toArray(new String[0]);
+ }
+
+ /**
+ * Given a comma-delimited list of integers, grab and parse the first int.
+ *
+ * @param str
+ * a comma-delimited list of values, the first of which is an
+ * int.
+ * @return the first field in the string, cast to int
+ */
+ private int getFirstInt(String str) {
+ String[] parts = str.split(",");
+ return Integer.parseInt(parts[0]);
+ }
+
+ public void runWhereTest(String whereClause, String firstValStr,
+ int numExpectedResults, int expectedSum) throws IOException {
+
+ String[] columns = MSSQLTestUtils.getColumns();
+ ClassLoader prevClassLoader = null;
+ SequenceFile.Reader reader = null;
+
+ String[] argv = getArgv(true, columns, whereClause);
+ runImport(argv);
+ try {
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+
+ SqoopOptions opts = new ImportTool().parseArguments(getArgv(false,
+ columns, whereClause), null, null, true);
+ opts.setUsername(username);
+ opts.setPassword(password);
+ CompilationManager compileMgr = new CompilationManager(opts);
+ String jarFileName = compileMgr.getJarFilename();
+
+ prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
+ getTableName());
+
+ reader = SeqFileReader.getSeqFileReader(getDataFilePath()
+ .toString());
+
+ // here we can actually instantiate (k, v) pairs.
+ Configuration conf = new Configuration();
+ Object key = ReflectionUtils
+ .newInstance(reader.getKeyClass(), conf);
+ Object val = ReflectionUtils.newInstance(reader.getValueClass(),
+ conf);
+
+ if (reader.next(key) == null) {
+ fail("Empty SequenceFile during import");
+ }
+
+ // make sure that the value we think should be at the top, is.
+ reader.getCurrentValue(val);
+ assertEquals("Invalid ordering within sorted SeqFile", firstValStr,
+ val.toString());
+
+ // We know that these values are two ints separated by a ','
+ // character.
+ // Since this is all dynamic, though, we don't want to actually link
+ // against the class and use its methods. So we just parse this back
+ // into int fields manually. Sum them up and ensure that we get the
+ // expected total for the first column, to verify that we got all
+ // the
+ // results from the db into the file.
+ int curSum = getFirstInt(val.toString());
+ int totalResults = 1;
+
+ // now sum up everything else in the file.
+ while (reader.next(key) != null) {
+ reader.getCurrentValue(val);
+ curSum += getFirstInt(val.toString());
+ totalResults++;
+ }
+
+ assertEquals("Total sum of first db column mismatch", expectedSum,
+ curSum);
+ assertEquals("Incorrect number of results for query",
+ numExpectedResults, totalResults);
+ } catch (InvalidOptionsException ioe) {
+ fail(ioe.toString());
+ } catch (ParseException pe) {
+ fail(pe.toString());
+ } finally {
+ IOUtils.closeStream(reader);
+
+ if (null != prevClassLoader) {
+ ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
+ }
+ }
+ }
+
+ @Test
+ public void testSingleClauseWhere() throws IOException {
+ String whereClause = "L_ORDERKEY > 0 ";
+ runWhereTest(whereClause,
+ "1,2,3,4,5,6.00,7.00,8.00,AB,CD,abcd,efgh,hijk,dothis,likethis,nocomments"
+ + "\n", 4, 10);
+ }
+
+ @Test
+ public void testMultiClauseWhere() throws IOException {
+ String whereClause = "L_ORDERKEY > 1 AND L_PARTKEY < 4";
+ runWhereTest(whereClause,
+ "2,3,4,5,6,7.00,8.00,9.00,AB,CD,abcd,efgh,hijk,dothis,likethis,nocomments"
+ + "\n", 1, 2);
+ }
+
+ protected boolean useHsqldbTestServer() {
+
+ return false;
+ }
+
+ protected String getConnectString() {
+ return MSSQLTestUtils.getDBConnectString();
+ }
+
+ /**
+ * Drop a table if it already exists in the database.
+ *
+ * @param table
+ * the name of the table to drop.
+ * @throws SQLException
+ * if something goes wrong.
+ */
+ protected void dropTableIfExists(String table) throws SQLException {
+ Connection conn = getManager().getConnection();
+ String sqlStmt = "IF OBJECT_ID('" + table
+ + "') IS NOT NULL DROP TABLE " + table;
+
+ PreparedStatement statement = conn.prepareStatement(sqlStmt,
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ statement.executeUpdate();
+ conn.commit();
+ } finally {
+ statement.close();
+ }
+ }
+
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ SqoopOptions opt = new SqoopOptions(conf);
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ SqoopOptions opts = new SqoopOptions(conf);
+ opts.setUsername(username);
+ opts.setPassword(password);
+
+ return opt;
+ }
+
+ protected String getTableName() {
+ return "tpch1m_lineitem";
+ }
+}
[4/4] sqoop git commit: SQOOP-3174: Add SQLServer manual tests to 3rd
party test suite
Posted by an...@apache.org.
SQOOP-3174: Add SQLServer manual tests to 3rd party test suite
(Boglarka Egyed via Anna Szonyi)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/558bdaea
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/558bdaea
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/558bdaea
Branch: refs/heads/trunk
Commit: 558bdaea907200bfdea2e6e434aabf97e7a566f2
Parents: 0ca73d4
Author: Anna Szonyi <an...@apache.org>
Authored: Mon May 15 10:03:49 2017 +0200
Committer: Anna Szonyi <an...@apache.org>
Committed: Mon May 15 10:03:49 2017 +0200
----------------------------------------------------------------------
build.xml | 2 +-
.../SQLServerManagerExportManualTest.java | 473 -----------
.../manager/SQLServerManagerExportTest.java | 473 +++++++++++
.../SQLServerManagerImportManualTest.java | 368 --------
.../manager/SQLServerManagerImportTest.java | 370 ++++++++
...erDatatypeExportDelimitedFileManualTest.java | 90 --
...QLServerDatatypeExportDelimitedFileTest.java | 92 ++
...verDatatypeExportSequenceFileManualTest.java | 262 ------
...SQLServerDatatypeExportSequenceFileTest.java | 264 ++++++
...erDatatypeImportDelimitedFileManualTest.java | 304 -------
...QLServerDatatypeImportDelimitedFileTest.java | 306 +++++++
...verDatatypeImportSequenceFileManualTest.java | 845 ------------------
...SQLServerDatatypeImportSequenceFileTest.java | 847 +++++++++++++++++++
.../SQLServerHiveImportManualTest.java | 190 -----
.../sqlserver/SQLServerHiveImportTest.java | 192 +++++
.../sqlserver/SQLServerManagerManualTest.java | 366 --------
.../manager/sqlserver/SQLServerManagerTest.java | 368 ++++++++
.../sqlserver/SQLServerMultiColsManualTest.java | 136 ---
.../sqlserver/SQLServerMultiColsTest.java | 138 +++
.../sqlserver/SQLServerMultiMapsManualTest.java | 320 -------
.../sqlserver/SQLServerMultiMapsTest.java | 322 +++++++
.../SQLServerParseMethodsManualTest.java | 285 -------
.../sqlserver/SQLServerParseMethodsTest.java | 287 +++++++
.../sqlserver/SQLServerQueryManualTest.java | 301 -------
.../manager/sqlserver/SQLServerQueryTest.java | 303 +++++++
.../sqlserver/SQLServerSplitByManualTest.java | 269 ------
.../manager/sqlserver/SQLServerSplitByTest.java | 271 ++++++
.../sqlserver/SQLServerWhereManualTest.java | 290 -------
.../manager/sqlserver/SQLServerWhereTest.java | 292 +++++++
29 files changed, 4526 insertions(+), 4500 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 10deb83..af43c47 100644
--- a/build.xml
+++ b/build.xml
@@ -743,7 +743,7 @@
<sysproperty key="test.build.data" value="${build.test}/data"/>
<sysproperty key="build.test" value="${build.test}"/>
- <!-- microsoft sqlserver manual test related properties-->
+ <!-- microsoft sqlserver thirdparty test related properties-->
<sysproperty key="test.data.dir" value="${basedir}/testdata"/>
<sysproperty key="ms.datatype.test.data.file.export" value="DatatypeTestData-export-lite.txt"/>
<sysproperty key="ms.datatype.test.data.file.import" value="DatatypeTestData-import-lite.txt"/>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/com/cloudera/sqoop/manager/SQLServerManagerExportManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/SQLServerManagerExportManualTest.java b/src/test/com/cloudera/sqoop/manager/SQLServerManagerExportManualTest.java
deleted file mode 100644
index 668a3a9..0000000
--- a/src/test/com/cloudera/sqoop/manager/SQLServerManagerExportManualTest.java
+++ /dev/null
@@ -1,473 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.sqoop.manager;
-
-import com.cloudera.sqoop.ConnFactory;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import com.cloudera.sqoop.testutil.ExportJobTestCase;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.Writer;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * Please see instructions in SQLServerManagerImportManualTest.
- */
-public class SQLServerManagerExportManualTest extends ExportJobTestCase {
-
- public static final Log LOG = LogFactory.getLog(
- SQLServerManagerExportManualTest.class.getName());
-
- static final String HOST_URL = System.getProperty(
- "sqoop.test.sqlserver.connectstring.host_url",
- "jdbc:sqlserver://sqlserverhost:1433");
- static final String DATABASE_NAME = System.getProperty(
- "sqoop.test.sqlserver.database",
- "sqooptest");
- static final String DATABASE_USER = System.getProperty(
- "ms.sqlserver.username",
- "sqoopuser");
- static final String DATABASE_PASSWORD = System.getProperty(
- "ms.sqlserver.password",
- "password");
-
- static final String SCHEMA_DBO = "dbo";
- static final String DBO_TABLE_NAME = "EMPLOYEES_MSSQL";
- static final String DBO_BINARY_TABLE_NAME = "BINARYTYPE_MSSQL";
- static final String SCHEMA_SCH = "sch";
- static final String SCH_TABLE_NAME = "PRIVATE_TABLE";
- static final String CONNECT_STRING = HOST_URL
- + ";databaseName=" + DATABASE_NAME;
-
- static final String CONNECTOR_FACTORY = System.getProperty(
- "sqoop.test.msserver.connector.factory",
- ConnFactory.DEFAULT_FACTORY_CLASS_NAMES);
-
- // instance variables populated during setUp, used during tests
- private SQLServerManager manager;
- private Configuration conf = new Configuration();
- private Connection conn = null;
-
- @Override
- protected Configuration getConf() {
- return conf;
- }
-
- @Override
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- private String getDropTableStatement(String schema, String tableName) {
- return "DROP TABLE IF EXISTS " + manager.escapeObjectName(schema)
- + "." + manager.escapeObjectName(tableName);
- }
-
- @Before
- public void setUp() {
- super.setUp();
-
- SqoopOptions options = new SqoopOptions(CONNECT_STRING,
- DBO_TABLE_NAME);
- options.setUsername(DATABASE_USER);
- options.setPassword(DATABASE_PASSWORD);
-
- manager = new SQLServerManager(options);
-
- createTableAndPopulateData(SCHEMA_DBO, DBO_TABLE_NAME);
- createTableAndPopulateData(SCHEMA_SCH, SCH_TABLE_NAME);
-
- // To test with Microsoft SQL server connector, copy the connector jar to
- // sqoop.thirdparty.lib.dir and set sqoop.test.msserver.connector.factory
- // to com.microsoft.sqoop.SqlServer.MSSQLServerManagerFactory. By default,
- // the built-in SQL server connector is used.
- conf.setStrings(ConnFactory.FACTORY_CLASS_NAMES_KEY, CONNECTOR_FACTORY);
- }
-
- public void createTableAndPopulateData(String schema, String table) {
- String fulltableName = manager.escapeObjectName(schema)
- + "." + manager.escapeObjectName(table);
-
- Statement stmt = null;
-
- // Create schema if needed
- try {
- conn = manager.getConnection();
- stmt = conn.createStatement();
- stmt.execute("CREATE SCHEMA " + schema);
- conn.commit();
- } catch (SQLException sqlE) {
- LOG.info("Can't create schema: " + sqlE.getMessage());
- } finally {
- try {
- if (null != stmt) {
- stmt.close();
- }
- } catch (Exception ex) {
- LOG.warn("Exception while closing stmt", ex);
- }
- }
-
- // Drop the existing table, if there is one.
- try {
- conn = manager.getConnection();
- stmt = conn.createStatement();
- stmt.execute("DROP TABLE " + fulltableName);
- conn.commit();
- } catch (SQLException sqlE) {
- LOG.info("Table was not dropped: " + sqlE.getMessage());
- } finally {
- try {
- if (null != stmt) {
- stmt.close();
- }
- } catch (Exception ex) {
- LOG.warn("Exception while closing stmt", ex);
- }
- }
-
- // Create and populate table
- try {
- conn = manager.getConnection();
- conn.setAutoCommit(false);
- stmt = conn.createStatement();
-
- // create the database table and populate it with data.
- stmt.executeUpdate("CREATE TABLE " + fulltableName + " ("
- + "id INT NOT NULL, "
- + "name VARCHAR(24) NOT NULL, "
- + "salary FLOAT, "
- + "dept VARCHAR(32), "
- + "PRIMARY KEY (id))");
- conn.commit();
- } catch (SQLException sqlE) {
- LOG.error("Encountered SQL Exception: ", sqlE);
- sqlE.printStackTrace();
- fail("SQLException when running test setUp(): " + sqlE);
- } finally {
- try {
- if (null != stmt) {
- stmt.close();
- }
- } catch (Exception ex) {
- LOG.warn("Exception while closing connection/stmt", ex);
- }
- }
- }
-
- public void createSQLServerBinaryTypeTable(String schema, String table) {
- String fulltableName = manager.escapeObjectName(schema)
- + "." + manager.escapeObjectName(table);
-
- Statement stmt = null;
-
- // Create schema if needed
- try {
- conn = manager.getConnection();
- stmt = conn.createStatement();
- stmt.execute("CREATE SCHEMA " + schema);
- conn.commit();
- } catch (SQLException sqlE) {
- LOG.info("Can't create schema: " + sqlE.getMessage());
- } finally {
- try {
- if (null != stmt) {
- stmt.close();
- }
- } catch (Exception ex) {
- LOG.warn("Exception while closing stmt", ex);
- }
- }
-
- // Drop the existing table, if there is one.
- try {
- conn = manager.getConnection();
- stmt = conn.createStatement();
- stmt.execute("DROP TABLE " + fulltableName);
- conn.commit();
- } catch (SQLException sqlE) {
- LOG.info("Table was not dropped: " + sqlE.getMessage());
- } finally {
- try {
- if (null != stmt) {
- stmt.close();
- }
- } catch (Exception ex) {
- LOG.warn("Exception while closing stmt", ex);
- }
- }
-
- // Create and populate table
- try {
- conn = manager.getConnection();
- conn.setAutoCommit(false);
- stmt = conn.createStatement();
-
- // create the database table and populate it with data.
- stmt.executeUpdate("CREATE TABLE " + fulltableName + " ("
- + "id INT PRIMARY KEY, "
- + "b1 BINARY(10), "
- + "b2 VARBINARY(10))");
- conn.commit();
- } catch (SQLException sqlE) {
- LOG.error("Encountered SQL Exception: ", sqlE);
- sqlE.printStackTrace();
- fail("SQLException when running test setUp(): " + sqlE);
- } finally {
- try {
- if (null != stmt) {
- stmt.close();
- }
- } catch (Exception ex) {
- LOG.warn("Exception while closing connection/stmt", ex);
- }
- }
- }
-
- @After
- public void tearDown() {
- try {
- Statement stmt = conn.createStatement();
- stmt.executeUpdate(getDropTableStatement(SCHEMA_DBO, DBO_TABLE_NAME));
- stmt.executeUpdate(getDropTableStatement(SCHEMA_SCH, SCH_TABLE_NAME));
- } catch (SQLException e) {
- LOG.error("Can't clean up the database:", e);
- }
-
- super.tearDown();
- try {
- conn.close();
- manager.close();
- } catch (SQLException sqlE) {
- LOG.error("Got SQLException: " + sqlE.toString());
- fail("Got SQLException: " + sqlE.toString());
- }
- }
-
- private String [] getArgv(String tableName,
- String... extraArgs) {
- ArrayList<String> args = new ArrayList<String>();
-
- CommonArgs.addHadoopFlags(args);
-
- args.add("--table");
- args.add(tableName);
- args.add("--export-dir");
- args.add(getWarehouseDir());
- args.add("--fields-terminated-by");
- args.add(",");
- args.add("--lines-terminated-by");
- args.add("\\n");
- args.add("--connect");
- args.add(CONNECT_STRING);
- args.add("--username");
- args.add(DATABASE_USER);
- args.add("--password");
- args.add(DATABASE_PASSWORD);
- args.add("-m");
- args.add("1");
-
- for (String arg : extraArgs) {
- args.add(arg);
- }
-
- return args.toArray(new String[0]);
- }
-
- protected void createTestFile(String filename,
- String[] lines)
- throws IOException {
- new File(getWarehouseDir()).mkdirs();
- File file = new File(getWarehouseDir() + "/" + filename);
- Writer output = new BufferedWriter(new FileWriter(file));
- for(String line : lines) {
- output.write(line);
- output.write("\n");
- }
- output.close();
- }
-
- @Test
- public void testExport() throws IOException, SQLException {
- createTestFile("inputFile", new String[] {
- "2,Bob,400,sales",
- "3,Fred,15,marketing",
- });
-
- runExport(getArgv(DBO_TABLE_NAME));
-
- assertRowCount(2, escapeObjectName(DBO_TABLE_NAME), conn);
- }
-
- @Test
- public void testExportCustomSchema() throws IOException, SQLException {
- createTestFile("inputFile", new String[] {
- "2,Bob,400,sales",
- "3,Fred,15,marketing",
- });
-
- String[] extra = new String[] {"--",
- "--schema",
- SCHEMA_SCH,
- };
-
- runExport(getArgv(SCH_TABLE_NAME, extra));
-
- assertRowCount(
- 2,
- escapeObjectName(SCHEMA_SCH) + "." + escapeObjectName(SCH_TABLE_NAME),
- conn
- );
- }
-
- @Test
- public void testExportTableHints() throws IOException, SQLException {
- createTestFile("inputFile", new String[] {
- "2,Bob,400,sales",
- "3,Fred,15,marketing",
- });
-
- String []extra = new String[] {"--", "--table-hints",
- "ROWLOCK",
- };
- runExport(getArgv(DBO_TABLE_NAME, extra));
- assertRowCount(2, escapeObjectName(DBO_TABLE_NAME), conn);
- }
-
- @Test
- public void testExportTableHintsMultiple() throws IOException, SQLException {
- createTestFile("inputFile", new String[] {
- "2,Bob,400,sales",
- "3,Fred,15,marketing",
- });
-
- String []extra = new String[] {"--", "--table-hints",
- "ROWLOCK,NOWAIT",
- };
- runExport(getArgv(DBO_TABLE_NAME, extra));
- assertRowCount(2, escapeObjectName(DBO_TABLE_NAME), conn);
- }
-
- @Test
- public void testSQLServerBinaryType() throws IOException, SQLException {
- createSQLServerBinaryTypeTable(SCHEMA_DBO, DBO_BINARY_TABLE_NAME);
- createTestFile("inputFile", new String[] {
- "1,73 65 63 72 65 74 00 00 00 00,73 65 63 72 65 74"
- });
- String[] expectedContent = {"73656372657400000000", "736563726574"};
- runExport(getArgv(DBO_BINARY_TABLE_NAME));
- assertRowCount(1, escapeObjectName(DBO_BINARY_TABLE_NAME), conn);
- checkSQLBinaryTableContent(expectedContent, escapeObjectName(DBO_BINARY_TABLE_NAME), conn);
- }
-
- /** Make sure mixed update/insert export work correctly. */
- @Test
- public void testUpsertTextExport() throws IOException, SQLException {
- createTestFile("inputFile", new String[] {
- "2,Bob,400,sales",
- "3,Fred,15,marketing",
- });
- // first time will be insert.
- runExport(getArgv(SCH_TABLE_NAME, "--update-key", "id",
- "--update-mode", "allowinsert", "--", "--schema", SCHEMA_SCH));
- // second time will be update.
- runExport(getArgv(SCH_TABLE_NAME, "--update-key", "id",
- "--update-mode", "allowinsert", "--", "--schema", SCHEMA_SCH));
- assertRowCount(2, escapeObjectName(SCHEMA_SCH) + "." + escapeObjectName(SCH_TABLE_NAME), conn);
- }
-
- public static void checkSQLBinaryTableContent(String[] expected, String tableName, Connection connection){
- Statement stmt = null;
- ResultSet rs = null;
- try {
- stmt = connection.createStatement();
- rs = stmt.executeQuery("SELECT TOP 1 [b1], [b2] FROM " + tableName);
- rs.next();
- assertEquals(expected[0], rs.getString("b1"));
- assertEquals(expected[1], rs.getString("b2"));
- } catch (SQLException e) {
- LOG.error("Can't verify table content", e);
- fail();
- } finally {
- try {
- connection.commit();
-
- if (stmt != null) {
- stmt.close();
- }
- if (rs != null) {
- rs.close();
- }
- } catch (SQLException ex) {
- LOG.info("Ignored exception in finally block.");
- }
- }
- }
-
- public static void assertRowCount(long expected,
- String tableName,
- Connection connection) {
- Statement stmt = null;
- ResultSet rs = null;
- try {
- stmt = connection.createStatement();
- rs = stmt.executeQuery("SELECT count(*) FROM " + tableName);
-
- rs.next();
-
- assertEquals(expected, rs.getLong(1));
- } catch (SQLException e) {
- LOG.error("Can't verify number of rows", e);
- fail();
- } finally {
- try {
- connection.commit();
-
- if (stmt != null) {
- stmt.close();
- }
- if (rs != null) {
- rs.close();
- }
- } catch (SQLException ex) {
- LOG.info("Ignored exception in finally block.");
- }
- }
- }
-
- public String escapeObjectName(String objectName) {
- return "[" + objectName + "]";
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/com/cloudera/sqoop/manager/SQLServerManagerExportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/SQLServerManagerExportTest.java b/src/test/com/cloudera/sqoop/manager/SQLServerManagerExportTest.java
new file mode 100644
index 0000000..c87994f
--- /dev/null
+++ b/src/test/com/cloudera/sqoop/manager/SQLServerManagerExportTest.java
@@ -0,0 +1,473 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.cloudera.sqoop.manager;
+
+import com.cloudera.sqoop.ConnFactory;
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.testutil.CommonArgs;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import com.cloudera.sqoop.testutil.ExportJobTestCase;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.Writer;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Please see instructions in SQLServerManagerImportTest.
+ */
+public class SQLServerManagerExportTest extends ExportJobTestCase {
+
+ public static final Log LOG = LogFactory.getLog(
+ SQLServerManagerExportTest.class.getName());
+
+ static final String HOST_URL = System.getProperty(
+ "sqoop.test.sqlserver.connectstring.host_url",
+ "jdbc:sqlserver://sqlserverhost:1433");
+ static final String DATABASE_NAME = System.getProperty(
+ "sqoop.test.sqlserver.database",
+ "sqooptest");
+ static final String DATABASE_USER = System.getProperty(
+ "ms.sqlserver.username",
+ "sqoopuser");
+ static final String DATABASE_PASSWORD = System.getProperty(
+ "ms.sqlserver.password",
+ "password");
+
+ static final String SCHEMA_DBO = "dbo";
+ static final String DBO_TABLE_NAME = "EMPLOYEES_MSSQL";
+ static final String DBO_BINARY_TABLE_NAME = "BINARYTYPE_MSSQL";
+ static final String SCHEMA_SCH = "sch";
+ static final String SCH_TABLE_NAME = "PRIVATE_TABLE";
+ static final String CONNECT_STRING = HOST_URL
+ + ";databaseName=" + DATABASE_NAME;
+
+ static final String CONNECTOR_FACTORY = System.getProperty(
+ "sqoop.test.msserver.connector.factory",
+ ConnFactory.DEFAULT_FACTORY_CLASS_NAMES);
+
+ // instance variables populated during setUp, used during tests
+ private SQLServerManager manager;
+ private Configuration conf = new Configuration();
+ private Connection conn = null;
+
+ @Override
+ protected Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ private String getDropTableStatement(String schema, String tableName) {
+ return "DROP TABLE IF EXISTS " + manager.escapeObjectName(schema)
+ + "." + manager.escapeObjectName(tableName);
+ }
+
+ @Before
+ public void setUp() {
+ super.setUp();
+
+ SqoopOptions options = new SqoopOptions(CONNECT_STRING,
+ DBO_TABLE_NAME);
+ options.setUsername(DATABASE_USER);
+ options.setPassword(DATABASE_PASSWORD);
+
+ manager = new SQLServerManager(options);
+
+ createTableAndPopulateData(SCHEMA_DBO, DBO_TABLE_NAME);
+ createTableAndPopulateData(SCHEMA_SCH, SCH_TABLE_NAME);
+
+ // To test with Microsoft SQL server connector, copy the connector jar to
+ // sqoop.thirdparty.lib.dir and set sqoop.test.msserver.connector.factory
+ // to com.microsoft.sqoop.SqlServer.MSSQLServerManagerFactory. By default,
+ // the built-in SQL server connector is used.
+ conf.setStrings(ConnFactory.FACTORY_CLASS_NAMES_KEY, CONNECTOR_FACTORY);
+ }
+
+ public void createTableAndPopulateData(String schema, String table) {
+ String fulltableName = manager.escapeObjectName(schema)
+ + "." + manager.escapeObjectName(table);
+
+ Statement stmt = null;
+
+ // Create schema if needed
+ try {
+ conn = manager.getConnection();
+ stmt = conn.createStatement();
+ stmt.execute("CREATE SCHEMA " + schema);
+ conn.commit();
+ } catch (SQLException sqlE) {
+ LOG.info("Can't create schema: " + sqlE.getMessage());
+ } finally {
+ try {
+ if (null != stmt) {
+ stmt.close();
+ }
+ } catch (Exception ex) {
+ LOG.warn("Exception while closing stmt", ex);
+ }
+ }
+
+ // Drop the existing table, if there is one.
+ try {
+ conn = manager.getConnection();
+ stmt = conn.createStatement();
+ stmt.execute("DROP TABLE " + fulltableName);
+ conn.commit();
+ } catch (SQLException sqlE) {
+ LOG.info("Table was not dropped: " + sqlE.getMessage());
+ } finally {
+ try {
+ if (null != stmt) {
+ stmt.close();
+ }
+ } catch (Exception ex) {
+ LOG.warn("Exception while closing stmt", ex);
+ }
+ }
+
+ // Create and populate table
+ try {
+ conn = manager.getConnection();
+ conn.setAutoCommit(false);
+ stmt = conn.createStatement();
+
+ // create the database table and populate it with data.
+ stmt.executeUpdate("CREATE TABLE " + fulltableName + " ("
+ + "id INT NOT NULL, "
+ + "name VARCHAR(24) NOT NULL, "
+ + "salary FLOAT, "
+ + "dept VARCHAR(32), "
+ + "PRIMARY KEY (id))");
+ conn.commit();
+ } catch (SQLException sqlE) {
+ LOG.error("Encountered SQL Exception: ", sqlE);
+ sqlE.printStackTrace();
+ fail("SQLException when running test setUp(): " + sqlE);
+ } finally {
+ try {
+ if (null != stmt) {
+ stmt.close();
+ }
+ } catch (Exception ex) {
+ LOG.warn("Exception while closing connection/stmt", ex);
+ }
+ }
+ }
+
+ public void createSQLServerBinaryTypeTable(String schema, String table) {
+ String fulltableName = manager.escapeObjectName(schema)
+ + "." + manager.escapeObjectName(table);
+
+ Statement stmt = null;
+
+ // Create schema if needed
+ try {
+ conn = manager.getConnection();
+ stmt = conn.createStatement();
+ stmt.execute("CREATE SCHEMA " + schema);
+ conn.commit();
+ } catch (SQLException sqlE) {
+ LOG.info("Can't create schema: " + sqlE.getMessage());
+ } finally {
+ try {
+ if (null != stmt) {
+ stmt.close();
+ }
+ } catch (Exception ex) {
+ LOG.warn("Exception while closing stmt", ex);
+ }
+ }
+
+ // Drop the existing table, if there is one.
+ try {
+ conn = manager.getConnection();
+ stmt = conn.createStatement();
+ stmt.execute("DROP TABLE " + fulltableName);
+ conn.commit();
+ } catch (SQLException sqlE) {
+ LOG.info("Table was not dropped: " + sqlE.getMessage());
+ } finally {
+ try {
+ if (null != stmt) {
+ stmt.close();
+ }
+ } catch (Exception ex) {
+ LOG.warn("Exception while closing stmt", ex);
+ }
+ }
+
+ // Create and populate table
+ try {
+ conn = manager.getConnection();
+ conn.setAutoCommit(false);
+ stmt = conn.createStatement();
+
+ // create the database table and populate it with data.
+ stmt.executeUpdate("CREATE TABLE " + fulltableName + " ("
+ + "id INT PRIMARY KEY, "
+ + "b1 BINARY(10), "
+ + "b2 VARBINARY(10))");
+ conn.commit();
+ } catch (SQLException sqlE) {
+ LOG.error("Encountered SQL Exception: ", sqlE);
+ sqlE.printStackTrace();
+ fail("SQLException when running test setUp(): " + sqlE);
+ } finally {
+ try {
+ if (null != stmt) {
+ stmt.close();
+ }
+ } catch (Exception ex) {
+ LOG.warn("Exception while closing connection/stmt", ex);
+ }
+ }
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.executeUpdate(getDropTableStatement(SCHEMA_DBO, DBO_TABLE_NAME));
+ stmt.executeUpdate(getDropTableStatement(SCHEMA_SCH, SCH_TABLE_NAME));
+ } catch (SQLException e) {
+ LOG.error("Can't clean up the database:", e);
+ }
+
+ super.tearDown();
+ try {
+ conn.close();
+ manager.close();
+ } catch (SQLException sqlE) {
+ LOG.error("Got SQLException: " + sqlE.toString());
+ fail("Got SQLException: " + sqlE.toString());
+ }
+ }
+
+ private String [] getArgv(String tableName,
+ String... extraArgs) {
+ ArrayList<String> args = new ArrayList<String>();
+
+ CommonArgs.addHadoopFlags(args);
+
+ args.add("--table");
+ args.add(tableName);
+ args.add("--export-dir");
+ args.add(getWarehouseDir());
+ args.add("--fields-terminated-by");
+ args.add(",");
+ args.add("--lines-terminated-by");
+ args.add("\\n");
+ args.add("--connect");
+ args.add(CONNECT_STRING);
+ args.add("--username");
+ args.add(DATABASE_USER);
+ args.add("--password");
+ args.add(DATABASE_PASSWORD);
+ args.add("-m");
+ args.add("1");
+
+ for (String arg : extraArgs) {
+ args.add(arg);
+ }
+
+ return args.toArray(new String[0]);
+ }
+
+ protected void createTestFile(String filename,
+ String[] lines)
+ throws IOException {
+ new File(getWarehouseDir()).mkdirs();
+ File file = new File(getWarehouseDir() + "/" + filename);
+ Writer output = new BufferedWriter(new FileWriter(file));
+ for(String line : lines) {
+ output.write(line);
+ output.write("\n");
+ }
+ output.close();
+ }
+
+ @Test
+ public void testExport() throws IOException, SQLException {
+ createTestFile("inputFile", new String[] {
+ "2,Bob,400,sales",
+ "3,Fred,15,marketing",
+ });
+
+ runExport(getArgv(DBO_TABLE_NAME));
+
+ assertRowCount(2, escapeObjectName(DBO_TABLE_NAME), conn);
+ }
+
+ @Test
+ public void testExportCustomSchema() throws IOException, SQLException {
+ createTestFile("inputFile", new String[] {
+ "2,Bob,400,sales",
+ "3,Fred,15,marketing",
+ });
+
+ String[] extra = new String[] {"--",
+ "--schema",
+ SCHEMA_SCH,
+ };
+
+ runExport(getArgv(SCH_TABLE_NAME, extra));
+
+ assertRowCount(
+ 2,
+ escapeObjectName(SCHEMA_SCH) + "." + escapeObjectName(SCH_TABLE_NAME),
+ conn
+ );
+ }
+
+ @Test
+ public void testExportTableHints() throws IOException, SQLException {
+ createTestFile("inputFile", new String[] {
+ "2,Bob,400,sales",
+ "3,Fred,15,marketing",
+ });
+
+ String []extra = new String[] {"--", "--table-hints",
+ "ROWLOCK",
+ };
+ runExport(getArgv(DBO_TABLE_NAME, extra));
+ assertRowCount(2, escapeObjectName(DBO_TABLE_NAME), conn);
+ }
+
+ @Test
+ public void testExportTableHintsMultiple() throws IOException, SQLException {
+ createTestFile("inputFile", new String[] {
+ "2,Bob,400,sales",
+ "3,Fred,15,marketing",
+ });
+
+ String []extra = new String[] {"--", "--table-hints",
+ "ROWLOCK,NOWAIT",
+ };
+ runExport(getArgv(DBO_TABLE_NAME, extra));
+ assertRowCount(2, escapeObjectName(DBO_TABLE_NAME), conn);
+ }
+
+ @Test
+ public void testSQLServerBinaryType() throws IOException, SQLException {
+ createSQLServerBinaryTypeTable(SCHEMA_DBO, DBO_BINARY_TABLE_NAME);
+ createTestFile("inputFile", new String[] {
+ "1,73 65 63 72 65 74 00 00 00 00,73 65 63 72 65 74"
+ });
+ String[] expectedContent = {"73656372657400000000", "736563726574"};
+ runExport(getArgv(DBO_BINARY_TABLE_NAME));
+ assertRowCount(1, escapeObjectName(DBO_BINARY_TABLE_NAME), conn);
+ checkSQLBinaryTableContent(expectedContent, escapeObjectName(DBO_BINARY_TABLE_NAME), conn);
+ }
+
+ /** Make sure mixed update/insert export work correctly. */
+ @Test
+ public void testUpsertTextExport() throws IOException, SQLException {
+ createTestFile("inputFile", new String[] {
+ "2,Bob,400,sales",
+ "3,Fred,15,marketing",
+ });
+ // first time will be insert.
+ runExport(getArgv(SCH_TABLE_NAME, "--update-key", "id",
+ "--update-mode", "allowinsert", "--", "--schema", SCHEMA_SCH));
+ // second time will be update.
+ runExport(getArgv(SCH_TABLE_NAME, "--update-key", "id",
+ "--update-mode", "allowinsert", "--", "--schema", SCHEMA_SCH));
+ assertRowCount(2, escapeObjectName(SCHEMA_SCH) + "." + escapeObjectName(SCH_TABLE_NAME), conn);
+ }
+
+ public static void checkSQLBinaryTableContent(String[] expected, String tableName, Connection connection){
+ Statement stmt = null;
+ ResultSet rs = null;
+ try {
+ stmt = connection.createStatement();
+ rs = stmt.executeQuery("SELECT TOP 1 [b1], [b2] FROM " + tableName);
+ rs.next();
+ assertEquals(expected[0], rs.getString("b1"));
+ assertEquals(expected[1], rs.getString("b2"));
+ } catch (SQLException e) {
+ LOG.error("Can't verify table content", e);
+ fail();
+ } finally {
+ try {
+ connection.commit();
+
+ if (stmt != null) {
+ stmt.close();
+ }
+ if (rs != null) {
+ rs.close();
+ }
+ } catch (SQLException ex) {
+ LOG.info("Ignored exception in finally block.");
+ }
+ }
+ }
+
+ public static void assertRowCount(long expected,
+ String tableName,
+ Connection connection) {
+ Statement stmt = null;
+ ResultSet rs = null;
+ try {
+ stmt = connection.createStatement();
+ rs = stmt.executeQuery("SELECT count(*) FROM " + tableName);
+
+ rs.next();
+
+ assertEquals(expected, rs.getLong(1));
+ } catch (SQLException e) {
+ LOG.error("Can't verify number of rows", e);
+ fail();
+ } finally {
+ try {
+ connection.commit();
+
+ if (stmt != null) {
+ stmt.close();
+ }
+ if (rs != null) {
+ rs.close();
+ }
+ } catch (SQLException ex) {
+ LOG.info("Ignored exception in finally block.");
+ }
+ }
+ }
+
+ public String escapeObjectName(String objectName) {
+ return "[" + objectName + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/com/cloudera/sqoop/manager/SQLServerManagerImportManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/SQLServerManagerImportManualTest.java b/src/test/com/cloudera/sqoop/manager/SQLServerManagerImportManualTest.java
deleted file mode 100644
index abc0479..0000000
--- a/src/test/com/cloudera/sqoop/manager/SQLServerManagerImportManualTest.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.manager;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.sqoop.ConnFactory;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
-import com.cloudera.sqoop.util.FileListing;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Test the SQLServerManager implementation.
- *
- * This uses JDBC to import data from an SQLServer database into HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerManagerImportManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerManagerImportManualTest extends ImportJobTestCase {
-
- public static final Log LOG = LogFactory.getLog(
- SQLServerManagerImportManualTest.class.getName());
-
- static final String HOST_URL = System.getProperty(
- "sqoop.test.sqlserver.connectstring.host_url",
- "jdbc:sqlserver://sqlserverhost:1433");
- static final String DATABASE_NAME = System.getProperty(
- "sqoop.test.sqlserver.database",
- "sqooptest");
- static final String DATABASE_USER = System.getProperty(
- "ms.sqlserver.username",
- "sqoopuser");
- static final String DATABASE_PASSWORD = System.getProperty(
- "ms.sqlserver.password",
- "password");
-
- static final String SCHEMA_DBO = "dbo";
- static final String DBO_TABLE_NAME = "EMPLOYEES_MSSQL";
- static final String SCHEMA_SCH = "sch";
- static final String SCH_TABLE_NAME = "PRIVATE_TABLE";
- static final String CONNECT_STRING = HOST_URL
- + ";databaseName=" + DATABASE_NAME;
-
- static final String CONNECTOR_FACTORY = System.getProperty(
- "sqoop.test.msserver.connector.factory",
- ConnFactory.DEFAULT_FACTORY_CLASS_NAMES);
-
- // instance variables populated during setUp, used during tests
- private SQLServerManager manager;
-
- private Configuration conf = new Configuration();
- private Connection conn = null;
-
- @Override
- protected Configuration getConf() {
- return conf;
- }
-
- @Override
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- private String getDropTableStatement(String schema, String tableName) {
- return "DROP TABLE IF EXISTS " + manager.escapeObjectName(schema)
- + "." + manager.escapeObjectName(tableName);
- }
-
- @Before
- public void setUp() {
- super.setUp();
-
- SqoopOptions options = new SqoopOptions(CONNECT_STRING,
- DBO_TABLE_NAME);
- options.setUsername(DATABASE_USER);
- options.setPassword(DATABASE_PASSWORD);
-
- manager = new SQLServerManager(options);
-
- createTableAndPopulateData(SCHEMA_DBO, DBO_TABLE_NAME);
- createTableAndPopulateData(SCHEMA_SCH, SCH_TABLE_NAME);
-
- // To test with Microsoft SQL server connector, copy the connector jar to
- // sqoop.thirdparty.lib.dir and set sqoop.test.msserver.connector.factory
- // to com.microsoft.sqoop.SqlServer.MSSQLServerManagerFactory. By default,
- // the built-in SQL server connector is used.
- conf.setStrings(ConnFactory.FACTORY_CLASS_NAMES_KEY, CONNECTOR_FACTORY);
- }
-
- public void createTableAndPopulateData(String schema, String table) {
- String fulltableName = manager.escapeObjectName(schema)
- + "." + manager.escapeObjectName(table);
-
- Statement stmt = null;
-
- // Create schema if needed
- try {
- conn = manager.getConnection();
- stmt = conn.createStatement();
- stmt.execute("CREATE SCHEMA " + schema);
- } catch (SQLException sqlE) {
- LOG.info("Can't create schema: " + sqlE.getMessage());
- } finally {
- try {
- if (null != stmt) {
- stmt.close();
- }
- } catch (Exception ex) {
- LOG.warn("Exception while closing stmt", ex);
- }
- }
-
- // Drop the existing table, if there is one.
- try {
- conn = manager.getConnection();
- stmt = conn.createStatement();
- stmt.execute("DROP TABLE " + fulltableName);
- } catch (SQLException sqlE) {
- LOG.info("Table was not dropped: " + sqlE.getMessage());
- } finally {
- try {
- if (null != stmt) {
- stmt.close();
- }
- } catch (Exception ex) {
- LOG.warn("Exception while closing stmt", ex);
- }
- }
-
- // Create and populate table
- try {
- conn = manager.getConnection();
- conn.setAutoCommit(false);
- stmt = conn.createStatement();
-
- // create the database table and populate it with data.
- stmt.executeUpdate("CREATE TABLE " + fulltableName + " ("
- + "id INT NOT NULL, "
- + "name VARCHAR(24) NOT NULL, "
- + "salary FLOAT, "
- + "dept VARCHAR(32), "
- + "PRIMARY KEY (id))");
-
- stmt.executeUpdate("INSERT INTO " + fulltableName + " VALUES("
- + "1,'Aaron', "
- + "1000000.00,'engineering')");
- stmt.executeUpdate("INSERT INTO " + fulltableName + " VALUES("
- + "2,'Bob', "
- + "400.00,'sales')");
- stmt.executeUpdate("INSERT INTO " + fulltableName + " VALUES("
- + "3,'Fred', 15.00,"
- + "'marketing')");
- conn.commit();
- } catch (SQLException sqlE) {
- LOG.error("Encountered SQL Exception: ", sqlE);
- sqlE.printStackTrace();
- fail("SQLException when running test setUp(): " + sqlE);
- } finally {
- try {
- if (null != stmt) {
- stmt.close();
- }
- } catch (Exception ex) {
- LOG.warn("Exception while closing connection/stmt", ex);
- }
- }
- }
-
- @After
- public void tearDown() {
- try {
- Statement stmt = conn.createStatement();
- stmt.executeUpdate(getDropTableStatement(SCHEMA_DBO, DBO_TABLE_NAME));
- stmt.executeUpdate(getDropTableStatement(SCHEMA_SCH, SCH_TABLE_NAME));
- } catch (SQLException e) {
- LOG.error("Can't clean up the database:", e);
- }
-
- super.tearDown();
- try {
- manager.close();
- } catch (SQLException sqlE) {
- LOG.error("Got SQLException: " + sqlE.toString());
- fail("Got SQLException: " + sqlE.toString());
- }
- }
-
- @Test
- public void testImportSimple() throws IOException {
- String [] expectedResults = {
- "1,Aaron,1000000.0,engineering",
- "2,Bob,400.0,sales",
- "3,Fred,15.0,marketing",
- };
-
- doImportAndVerify(DBO_TABLE_NAME, expectedResults);
- }
-
- @Test
- public void testImportExplicitDefaultSchema() throws IOException {
- String [] expectedResults = {
- "1,Aaron,1000000.0,engineering",
- "2,Bob,400.0,sales",
- "3,Fred,15.0,marketing",
- };
-
- String[] extraArgs = new String[] {"--schema", SCHEMA_DBO};
-
- doImportAndVerify(DBO_TABLE_NAME, expectedResults, extraArgs);
- }
-
- @Test
- public void testImportDifferentSchema() throws IOException {
- String [] expectedResults = {
- "1,Aaron,1000000.0,engineering",
- "2,Bob,400.0,sales",
- "3,Fred,15.0,marketing",
- };
-
- String[] extraArgs = new String[] {"--schema", SCHEMA_SCH};
-
- doImportAndVerify(SCH_TABLE_NAME, expectedResults, extraArgs);
- }
-
- @Test
- public void testImportTableHints() throws IOException {
- String [] expectedResults = {
- "1,Aaron,1000000.0,engineering",
- "2,Bob,400.0,sales",
- "3,Fred,15.0,marketing",
- };
-
- String[] extraArgs = new String[] {"--table-hints", "NOLOCK"};
- doImportAndVerify(DBO_TABLE_NAME, expectedResults, extraArgs);
- }
-
- @Test
- public void testImportTableHintsMultiple() throws IOException {
- String [] expectedResults = {
- "1,Aaron,1000000.0,engineering",
- "2,Bob,400.0,sales",
- "3,Fred,15.0,marketing",
- };
-
- String[] extraArgs = new String[] {"--table-hints", "NOLOCK,NOWAIT"};
- doImportAndVerify(DBO_TABLE_NAME, expectedResults, extraArgs);
- }
-
- private String [] getArgv(String tableName, String ... extraArgs) {
- ArrayList<String> args = new ArrayList<String>();
-
- CommonArgs.addHadoopFlags(args);
-
- args.add("--table");
- args.add(tableName);
- args.add("--warehouse-dir");
- args.add(getWarehouseDir());
- args.add("--connect");
- args.add(CONNECT_STRING);
- args.add("--username");
- args.add(DATABASE_USER);
- args.add("--password");
- args.add(DATABASE_PASSWORD);
- args.add("--num-mappers");
- args.add("1");
-
- if (extraArgs.length > 0) {
- args.add("--");
- for (String arg : extraArgs) {
- args.add(arg);
- }
- }
-
- return args.toArray(new String[0]);
- }
-
- private void doImportAndVerify(String tableName,
- String [] expectedResults,
- String ... extraArgs) throws IOException {
-
- Path warehousePath = new Path(this.getWarehouseDir());
- Path tablePath = new Path(warehousePath, tableName);
- Path filePath = new Path(tablePath, "part-m-00000");
-
- File tableFile = new File(tablePath.toString());
- if (tableFile.exists() && tableFile.isDirectory()) {
- // remove the directory before running the import.
- FileListing.recursiveDeleteDir(tableFile);
- }
-
- String [] argv = getArgv(tableName, extraArgs);
- try {
- runImport(argv);
- } catch (IOException ioe) {
- LOG.error("Got IOException during import: " + ioe.toString());
- ioe.printStackTrace();
- fail(ioe.toString());
- }
-
- File f = new File(filePath.toString());
- assertTrue("Could not find imported data file", f.exists());
- BufferedReader r = null;
- try {
- // Read through the file and make sure it's all there.
- r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
- for (String expectedLine : expectedResults) {
- assertEquals(expectedLine, r.readLine());
- }
- } catch (IOException ioe) {
- LOG.error("Got IOException verifying results: " + ioe.toString());
- ioe.printStackTrace();
- fail(ioe.toString());
- } finally {
- IOUtils.closeStream(r);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/com/cloudera/sqoop/manager/SQLServerManagerImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/SQLServerManagerImportTest.java b/src/test/com/cloudera/sqoop/manager/SQLServerManagerImportTest.java
new file mode 100644
index 0000000..714a592
--- /dev/null
+++ b/src/test/com/cloudera/sqoop/manager/SQLServerManagerImportTest.java
@@ -0,0 +1,370 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.cloudera.sqoop.manager;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.sqoop.ConnFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.testutil.CommonArgs;
+import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import com.cloudera.sqoop.util.FileListing;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test the SQLServerManager implementation.
+ *
+ * This uses JDBC to import data from an SQLServer database into HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerManagerImportTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerManagerImportTest extends ImportJobTestCase {
+
+ public static final Log LOG = LogFactory.getLog(
+ SQLServerManagerImportTest.class.getName());
+
+ static final String HOST_URL = System.getProperty(
+ "sqoop.test.sqlserver.connectstring.host_url",
+ "jdbc:sqlserver://sqlserverhost:1433");
+ static final String DATABASE_NAME = System.getProperty(
+ "sqoop.test.sqlserver.database",
+ "sqooptest");
+ static final String DATABASE_USER = System.getProperty(
+ "ms.sqlserver.username",
+ "sqoopuser");
+ static final String DATABASE_PASSWORD = System.getProperty(
+ "ms.sqlserver.password",
+ "password");
+
+ static final String SCHEMA_DBO = "dbo";
+ static final String DBO_TABLE_NAME = "EMPLOYEES_MSSQL";
+ static final String SCHEMA_SCH = "sch";
+ static final String SCH_TABLE_NAME = "PRIVATE_TABLE";
+ static final String CONNECT_STRING = HOST_URL
+ + ";databaseName=" + DATABASE_NAME;
+
+ static final String CONNECTOR_FACTORY = System.getProperty(
+ "sqoop.test.msserver.connector.factory",
+ ConnFactory.DEFAULT_FACTORY_CLASS_NAMES);
+
+ // instance variables populated during setUp, used during tests
+ private SQLServerManager manager;
+
+ private Configuration conf = new Configuration();
+ private Connection conn = null;
+
+ @Override
+ protected Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ private String getDropTableStatement(String schema, String tableName) {
+ return "DROP TABLE IF EXISTS " + manager.escapeObjectName(schema)
+ + "." + manager.escapeObjectName(tableName);
+ }
+
+ @Before
+ public void setUp() {
+ super.setUp();
+
+ SqoopOptions options = new SqoopOptions(CONNECT_STRING,
+ DBO_TABLE_NAME);
+ options.setUsername(DATABASE_USER);
+ options.setPassword(DATABASE_PASSWORD);
+
+ manager = new SQLServerManager(options);
+
+ createTableAndPopulateData(SCHEMA_DBO, DBO_TABLE_NAME);
+ createTableAndPopulateData(SCHEMA_SCH, SCH_TABLE_NAME);
+
+ // To test with Microsoft SQL server connector, copy the connector jar to
+ // sqoop.thirdparty.lib.dir and set sqoop.test.msserver.connector.factory
+ // to com.microsoft.sqoop.SqlServer.MSSQLServerManagerFactory. By default,
+ // the built-in SQL server connector is used.
+ conf.setStrings(ConnFactory.FACTORY_CLASS_NAMES_KEY, CONNECTOR_FACTORY);
+ }
+
+ public void createTableAndPopulateData(String schema, String table) {
+ String fulltableName = manager.escapeObjectName(schema)
+ + "." + manager.escapeObjectName(table);
+
+ Statement stmt = null;
+
+ // Create schema if needed
+ try {
+ conn = manager.getConnection();
+ stmt = conn.createStatement();
+ stmt.execute("CREATE SCHEMA " + schema);
+ } catch (SQLException sqlE) {
+ LOG.info("Can't create schema: " + sqlE.getMessage());
+ } finally {
+ try {
+ if (null != stmt) {
+ stmt.close();
+ }
+ } catch (Exception ex) {
+ LOG.warn("Exception while closing stmt", ex);
+ }
+ }
+
+ // Drop the existing table, if there is one.
+ try {
+ conn = manager.getConnection();
+ stmt = conn.createStatement();
+ stmt.execute("DROP TABLE " + fulltableName);
+ } catch (SQLException sqlE) {
+ LOG.info("Table was not dropped: " + sqlE.getMessage());
+ } finally {
+ try {
+ if (null != stmt) {
+ stmt.close();
+ }
+ } catch (Exception ex) {
+ LOG.warn("Exception while closing stmt", ex);
+ }
+ }
+
+ // Create and populate table
+ try {
+ conn = manager.getConnection();
+ conn.setAutoCommit(false);
+ stmt = conn.createStatement();
+
+ // create the database table and populate it with data.
+ stmt.executeUpdate("CREATE TABLE " + fulltableName + " ("
+ + "id INT NOT NULL, "
+ + "name VARCHAR(24) NOT NULL, "
+ + "salary FLOAT, "
+ + "dept VARCHAR(32), "
+ + "PRIMARY KEY (id))");
+
+ stmt.executeUpdate("INSERT INTO " + fulltableName + " VALUES("
+ + "1,'Aaron', "
+ + "1000000.00,'engineering')");
+ stmt.executeUpdate("INSERT INTO " + fulltableName + " VALUES("
+ + "2,'Bob', "
+ + "400.00,'sales')");
+ stmt.executeUpdate("INSERT INTO " + fulltableName + " VALUES("
+ + "3,'Fred', 15.00,"
+ + "'marketing')");
+ conn.commit();
+ } catch (SQLException sqlE) {
+ LOG.error("Encountered SQL Exception: ", sqlE);
+ sqlE.printStackTrace();
+ fail("SQLException when running test setUp(): " + sqlE);
+ } finally {
+ try {
+ if (null != stmt) {
+ stmt.close();
+ }
+ } catch (Exception ex) {
+ LOG.warn("Exception while closing connection/stmt", ex);
+ }
+ }
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.executeUpdate(getDropTableStatement(SCHEMA_DBO, DBO_TABLE_NAME));
+ stmt.executeUpdate(getDropTableStatement(SCHEMA_SCH, SCH_TABLE_NAME));
+ } catch (SQLException e) {
+ LOG.error("Can't clean up the database:", e);
+ }
+
+ super.tearDown();
+ try {
+ manager.close();
+ } catch (SQLException sqlE) {
+ LOG.error("Got SQLException: " + sqlE.toString());
+ fail("Got SQLException: " + sqlE.toString());
+ }
+ }
+
+ @Test
+ public void testImportSimple() throws IOException {
+ String [] expectedResults = {
+ "1,Aaron,1000000.0,engineering",
+ "2,Bob,400.0,sales",
+ "3,Fred,15.0,marketing",
+ };
+
+ doImportAndVerify(DBO_TABLE_NAME, expectedResults);
+ }
+
+ @Test
+ public void testImportExplicitDefaultSchema() throws IOException {
+ String [] expectedResults = {
+ "1,Aaron,1000000.0,engineering",
+ "2,Bob,400.0,sales",
+ "3,Fred,15.0,marketing",
+ };
+
+ String[] extraArgs = new String[] {"--schema", SCHEMA_DBO};
+
+ doImportAndVerify(DBO_TABLE_NAME, expectedResults, extraArgs);
+ }
+
+ @Test
+ public void testImportDifferentSchema() throws IOException {
+ String [] expectedResults = {
+ "1,Aaron,1000000.0,engineering",
+ "2,Bob,400.0,sales",
+ "3,Fred,15.0,marketing",
+ };
+
+ String[] extraArgs = new String[] {"--schema", SCHEMA_SCH};
+
+ doImportAndVerify(SCH_TABLE_NAME, expectedResults, extraArgs);
+ }
+
+ @Test
+ public void testImportTableHints() throws IOException {
+ String [] expectedResults = {
+ "1,Aaron,1000000.0,engineering",
+ "2,Bob,400.0,sales",
+ "3,Fred,15.0,marketing",
+ };
+
+ String[] extraArgs = new String[] {"--table-hints", "NOLOCK"};
+ doImportAndVerify(DBO_TABLE_NAME, expectedResults, extraArgs);
+ }
+
+ @Test
+ public void testImportTableHintsMultiple() throws IOException {
+ String [] expectedResults = {
+ "1,Aaron,1000000.0,engineering",
+ "2,Bob,400.0,sales",
+ "3,Fred,15.0,marketing",
+ };
+
+ String[] extraArgs = new String[] {"--table-hints", "NOLOCK,NOWAIT"};
+ doImportAndVerify(DBO_TABLE_NAME, expectedResults, extraArgs);
+ }
+
+ private String [] getArgv(String tableName, String ... extraArgs) {
+ ArrayList<String> args = new ArrayList<String>();
+
+ CommonArgs.addHadoopFlags(args);
+
+ args.add("--table");
+ args.add(tableName);
+ args.add("--warehouse-dir");
+ args.add(getWarehouseDir());
+ args.add("--connect");
+ args.add(CONNECT_STRING);
+ args.add("--username");
+ args.add(DATABASE_USER);
+ args.add("--password");
+ args.add(DATABASE_PASSWORD);
+ args.add("--num-mappers");
+ args.add("1");
+
+ if (extraArgs.length > 0) {
+ args.add("--");
+ for (String arg : extraArgs) {
+ args.add(arg);
+ }
+ }
+
+ return args.toArray(new String[0]);
+ }
+
+ private void doImportAndVerify(String tableName,
+ String [] expectedResults,
+ String ... extraArgs) throws IOException {
+
+ Path warehousePath = new Path(this.getWarehouseDir());
+ Path tablePath = new Path(warehousePath, tableName);
+ Path filePath = new Path(tablePath, "part-m-00000");
+
+ File tableFile = new File(tablePath.toString());
+ if (tableFile.exists() && tableFile.isDirectory()) {
+ // remove the directory before running the import.
+ FileListing.recursiveDeleteDir(tableFile);
+ }
+
+ String [] argv = getArgv(tableName, extraArgs);
+ try {
+ runImport(argv);
+ } catch (IOException ioe) {
+ LOG.error("Got IOException during import: " + ioe.toString());
+ ioe.printStackTrace();
+ fail(ioe.toString());
+ }
+
+ File f = new File(filePath.toString());
+ assertTrue("Could not find imported data file", f.exists());
+ BufferedReader r = null;
+ try {
+ // Read through the file and make sure it's all there.
+ r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
+ for (String expectedLine : expectedResults) {
+ assertEquals(expectedLine, r.readLine());
+ }
+ } catch (IOException ioe) {
+ LOG.error("Got IOException verifying results: " + ioe.toString());
+ ioe.printStackTrace();
+ fail(ioe.toString());
+ } finally {
+ IOUtils.closeStream(r);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportDelimitedFileManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportDelimitedFileManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportDelimitedFileManualTest.java
deleted file mode 100644
index 539eeb3..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportDelimitedFileManualTest.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.manager.sqlserver;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES;
-
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.io.BufferedWriter;
-
-/**
- * Test to export delimited file to SQL Server.
- *
- * This uses JDBC to export data to an SQLServer database from HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerDatatypeExportDelimitedFileManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerDatatypeExportDelimitedFileManualTest
- extends ManagerCompatExport {
-
- @Override
- public void createFile(DATATYPES dt, String[] data) throws IOException {
- Path tablePath = getTablePath(dt);
- Path filePath = new Path(tablePath, "part0000");
-
- Configuration conf = new Configuration();
- String hdfsroot;
- hdfsroot = System.getProperty("ms.datatype.test.hdfsprefix");
- if (hdfsroot == null) {
- hdfsroot = "hdfs://localhost/";
- }
- conf.set("fs.default.name", hdfsroot);
- FileSystem fs = FileSystem.get(conf);
- fs.mkdirs(tablePath);
- System.out.println("-----------------------------------Path : "
- + filePath);
- OutputStream os = fs.create(filePath);
-
- BufferedWriter w = new BufferedWriter(new OutputStreamWriter(os));
- for (int i = 0; i < data.length; i++) {
- w.write(data[i] + "\n");
- }
- w.close();
- os.close();
- }
-
- @Override
- public void createFile(DATATYPES dt, String data) throws IOException {
- createFile(dt, new String[] { data });
- }
-
- @Override
- public String getOutputFileName() {
- return "ManagerCompatExportDelim.txt";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportDelimitedFileTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportDelimitedFileTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportDelimitedFileTest.java
new file mode 100644
index 0000000..9b70af1
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportDelimitedFileTest.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.manager.sqlserver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES;
+
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.BufferedWriter;
+
+/**
+ * Test to export delimited file to SQL Server.
+ *
+ * This uses JDBC to export data to an SQLServer database from HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerDatatypeExportDelimitedFileTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerDatatypeExportDelimitedFileTest
+ extends ManagerCompatExport {
+
+ @Override
+ public void createFile(DATATYPES dt, String[] data) throws IOException {
+ Path tablePath = getTablePath(dt);
+ Path filePath = new Path(tablePath, "part0000");
+
+ Configuration conf = new Configuration();
+ String hdfsroot;
+ hdfsroot = System.getProperty("ms.datatype.test.hdfsprefix");
+ if (hdfsroot == null) {
+ hdfsroot = "hdfs://localhost/";
+ }
+ conf.set("fs.default.name", hdfsroot);
+ FileSystem fs = FileSystem.get(conf);
+ fs.mkdirs(tablePath);
+ System.out.println("-----------------------------------Path : "
+ + filePath);
+ OutputStream os = fs.create(filePath);
+
+ BufferedWriter w = new BufferedWriter(new OutputStreamWriter(os));
+ for (int i = 0; i < data.length; i++) {
+ w.write(data[i] + "\n");
+ }
+ w.close();
+ os.close();
+ }
+
+ @Override
+ public void createFile(DATATYPES dt, String data) throws IOException {
+ createFile(dt, new String[] { data });
+ }
+
+ @Override
+ public String getOutputFileName() {
+ return "ManagerCompatExportDelim.txt";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportSequenceFileManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportSequenceFileManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportSequenceFileManualTest.java
deleted file mode 100644
index 0f206d0..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportSequenceFileManualTest.java
+++ /dev/null
@@ -1,262 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.manager.sqlserver;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.lib.RecordParser;
-import com.cloudera.sqoop.lib.SqoopRecord;
-import com.cloudera.sqoop.tool.CodeGenTool;
-import com.cloudera.sqoop.util.ClassLoaderStack;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test to export sequence file to SQL Server.
- *
- * This uses JDBC to export data to an SQLServer database from HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerDatatypeExportSequenceFileManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerDatatypeExportSequenceFileManualTest
- extends ManagerCompatExport {
-
- private static Map jars = new HashMap();
-
- @Override
- public void createFile(DATATYPES dt, String[] data) throws Exception {
- try {
- codeGen(dt);
- // Instantiate the value record object via reflection.
- Class cls = Class.forName(getTableName(dt), true, Thread
- .currentThread().getContextClassLoader());
- SqoopRecord record = (SqoopRecord) ReflectionUtils.newInstance(cls,
- new Configuration());
-
- // Create the SequenceFile.
- Configuration conf = new Configuration();
- String hdfsroot;
- hdfsroot = System.getProperty("ms.datatype.test.hdfsprefix");
- if (hdfsroot == null){
- hdfsroot ="hdfs://localhost/";
- }
- conf.set("fs.default.name", hdfsroot);
- FileSystem fs = FileSystem.get(conf);
- Path tablePath = getTablePath(dt);
- Path filePath = new Path(tablePath, getTableName(dt));
- fs.mkdirs(tablePath);
- SequenceFile.Writer w = SequenceFile.createWriter(fs, conf,
- filePath, LongWritable.class, cls);
-
- int cnt = 0;
- for (String tmp : data) {
- record.parse(tmp + "\n");
- w.append(new LongWritable(cnt), record);
- }
-
- w.close();
- } catch (ClassNotFoundException cnfe) {
- throw new IOException(cnfe);
- } catch (RecordParser.ParseError pe) {
- throw new IOException(pe);
- }
- }
-
- @Override
- public void createFile(DATATYPES dt, String data) throws Exception {
- createFile(dt, new String[] { data });
- }
-
- public String[] codeGen(DATATYPES dt) throws Exception {
-
- CodeGenTool codeGen = new CodeGenTool();
-
- String[] codeGenArgs = getCodeGenArgv(dt);
- SqoopOptions options = codeGen.parseArguments(codeGenArgs, null, null,
- true);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
-
- options.setUsername(username);
- options.setPassword(password);
- codeGen.validateOptions(options);
-
- int ret = codeGen.run(options);
- assertEquals(0, ret);
- List<String> generatedJars = codeGen.getGeneratedJarFiles();
-
- assertNotNull(generatedJars);
- assertEquals("Expected 1 generated jar file", 1, generatedJars.size());
- String jarFileName = generatedJars.get(0);
- // Sqoop generates jars named "foo.jar"; by default, this should contain
- // a class named 'foo'. Extract the class name.
- Path jarPath = new Path(jarFileName);
- String jarBaseName = jarPath.getName();
- assertTrue(jarBaseName.endsWith(".jar"));
- assertTrue(jarBaseName.length() > ".jar".length());
- String className = jarBaseName.substring(0, jarBaseName.length()
- - ".jar".length());
-
- LOG.info("Using jar filename: " + jarFileName);
- LOG.info("Using class name: " + className);
-
- ClassLoader prevClassLoader = null;
-
-
- if (null != jarFileName) {
- prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
- className);
- System.out.println("Jar,class =" + jarFileName + " , "
- + className);
- }
-
- // Now run and verify the export.
- LOG.info("Exporting SequenceFile-based data");
- jars.put(dt, jarFileName);
- return (getArgv(dt, "--class-name", className, "--jar-file",
- jarFileName));
- }
-
- @Override
- protected String[] getArgv(DATATYPES dt) {
-
- String[] args = super.getArgv(dt);
- String[] addtionalArgs = Arrays.copyOf(args, args.length + 4);
-
- String[] additional = new String[4];
- additional[0] = "--class-name";
- additional[1] = getTableName(dt);
- additional[2] = "--jar-file";
- additional[3] = jars.get(dt).toString();
- for (int i = args.length, j = 0; i < addtionalArgs.length; i++, j++) {
- addtionalArgs[i] = additional[j];
- }
-
- for (String a : addtionalArgs) {
- System.out.println(a);
- }
- return addtionalArgs;
- }
-
- /**
- * @return an argv for the CodeGenTool to use when creating tables to
- * export.
- */
- protected String[] getCodeGenArgv(DATATYPES dt) {
- List<String> codeGenArgv = new ArrayList<String>();
-
- codeGenArgv.add("--table");
- codeGenArgv.add(getTableName(dt));
- codeGenArgv.add("--connect");
- codeGenArgv.add(MSSQLTestUtils.getDBConnectString());
- codeGenArgv.add("--fields-terminated-by");
- codeGenArgv.add("\\t");
- codeGenArgv.add("--lines-terminated-by");
- codeGenArgv.add("\\n");
-
- return codeGenArgv.toArray(new String[0]);
- }
-
- protected String[] getArgv(DATATYPES dt, String... additionalArgv) {
- ArrayList<String> args = new ArrayList<String>();
-
- // Any additional Hadoop flags (-D foo=bar) are prepended.
- if (null != additionalArgv) {
- boolean prevIsFlag = false;
- for (String arg : additionalArgv) {
- if (arg.equals("-D")) {
- args.add(arg);
- prevIsFlag = true;
- } else if (prevIsFlag) {
- args.add(arg);
- prevIsFlag = false;
- }
- }
- }
-
- // The sqoop-specific additional args are then added.
- if (null != additionalArgv) {
- boolean prevIsFlag = false;
- for (String arg : additionalArgv) {
- if (arg.equals("-D")) {
- prevIsFlag = true;
- continue;
- } else if (prevIsFlag) {
- prevIsFlag = false;
- continue;
- } else {
- // normal argument.
- args.add(arg);
- }
- }
- }
-
- args.add("--table");
- args.add(getTableName(dt));
- args.add("--export-dir");
- args.add(getTablePath(dt).toString());
- args.add("--connect");
- args.add(MSSQLTestUtils.getDBConnectString());
- args.add("--fields-terminated-by");
- args.add("\\t");
- args.add("--lines-terminated-by");
- args.add("\\n");
- args.add("-m");
- args.add("1");
-
- LOG.debug("args:");
- for (String a : args) {
- LOG.debug(" " + a);
- }
-
- return args.toArray(new String[0]);
- }
-
- public String getOutputFileName() {
- return "ManagerCompatExportSeq.txt";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportSequenceFileTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportSequenceFileTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportSequenceFileTest.java
new file mode 100644
index 0000000..a68ed30
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeExportSequenceFileTest.java
@@ -0,0 +1,264 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.manager.sqlserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.lib.RecordParser;
+import com.cloudera.sqoop.lib.SqoopRecord;
+import com.cloudera.sqoop.tool.CodeGenTool;
+import com.cloudera.sqoop.util.ClassLoaderStack;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test to export sequence file to SQL Server.
+ *
+ * This uses JDBC to export data to an SQLServer database from HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerDatatypeExportSequenceFileTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerDatatypeExportSequenceFileTest
+ extends ManagerCompatExport {
+
+ private static Map jars = new HashMap();
+
+ @Override
+ public void createFile(DATATYPES dt, String[] data) throws Exception {
+ try {
+ codeGen(dt);
+ // Instantiate the value record object via reflection.
+ Class cls = Class.forName(getTableName(dt), true, Thread
+ .currentThread().getContextClassLoader());
+ SqoopRecord record = (SqoopRecord) ReflectionUtils.newInstance(cls,
+ new Configuration());
+
+ // Create the SequenceFile.
+ Configuration conf = new Configuration();
+ String hdfsroot;
+ hdfsroot = System.getProperty("ms.datatype.test.hdfsprefix");
+ if (hdfsroot == null){
+ hdfsroot ="hdfs://localhost/";
+ }
+ conf.set("fs.default.name", hdfsroot);
+ FileSystem fs = FileSystem.get(conf);
+ Path tablePath = getTablePath(dt);
+ Path filePath = new Path(tablePath, getTableName(dt));
+ fs.mkdirs(tablePath);
+ SequenceFile.Writer w = SequenceFile.createWriter(fs, conf,
+ filePath, LongWritable.class, cls);
+
+ int cnt = 0;
+ for (String tmp : data) {
+ record.parse(tmp + "\n");
+ w.append(new LongWritable(cnt), record);
+ }
+
+ w.close();
+ } catch (ClassNotFoundException cnfe) {
+ throw new IOException(cnfe);
+ } catch (RecordParser.ParseError pe) {
+ throw new IOException(pe);
+ }
+ }
+
+ @Override
+ public void createFile(DATATYPES dt, String data) throws Exception {
+ createFile(dt, new String[] { data });
+ }
+
+ public String[] codeGen(DATATYPES dt) throws Exception {
+
+ CodeGenTool codeGen = new CodeGenTool();
+
+ String[] codeGenArgs = getCodeGenArgv(dt);
+ SqoopOptions options = codeGen.parseArguments(codeGenArgs, null, null,
+ true);
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+
+ options.setUsername(username);
+ options.setPassword(password);
+ codeGen.validateOptions(options);
+
+ int ret = codeGen.run(options);
+ assertEquals(0, ret);
+ List<String> generatedJars = codeGen.getGeneratedJarFiles();
+
+ assertNotNull(generatedJars);
+ assertEquals("Expected 1 generated jar file", 1, generatedJars.size());
+ String jarFileName = generatedJars.get(0);
+ // Sqoop generates jars named "foo.jar"; by default, this should contain
+ // a class named 'foo'. Extract the class name.
+ Path jarPath = new Path(jarFileName);
+ String jarBaseName = jarPath.getName();
+ assertTrue(jarBaseName.endsWith(".jar"));
+ assertTrue(jarBaseName.length() > ".jar".length());
+ String className = jarBaseName.substring(0, jarBaseName.length()
+ - ".jar".length());
+
+ LOG.info("Using jar filename: " + jarFileName);
+ LOG.info("Using class name: " + className);
+
+ ClassLoader prevClassLoader = null;
+
+
+ if (null != jarFileName) {
+ prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
+ className);
+ System.out.println("Jar,class =" + jarFileName + " , "
+ + className);
+ }
+
+ // Now run and verify the export.
+ LOG.info("Exporting SequenceFile-based data");
+ jars.put(dt, jarFileName);
+ return (getArgv(dt, "--class-name", className, "--jar-file",
+ jarFileName));
+ }
+
+ @Override
+ protected String[] getArgv(DATATYPES dt) {
+
+ String[] args = super.getArgv(dt);
+ String[] addtionalArgs = Arrays.copyOf(args, args.length + 4);
+
+ String[] additional = new String[4];
+ additional[0] = "--class-name";
+ additional[1] = getTableName(dt);
+ additional[2] = "--jar-file";
+ additional[3] = jars.get(dt).toString();
+ for (int i = args.length, j = 0; i < addtionalArgs.length; i++, j++) {
+ addtionalArgs[i] = additional[j];
+ }
+
+ for (String a : addtionalArgs) {
+ System.out.println(a);
+ }
+ return addtionalArgs;
+ }
+
+ /**
+ * @return an argv for the CodeGenTool to use when creating tables to
+ * export.
+ */
+ protected String[] getCodeGenArgv(DATATYPES dt) {
+ List<String> codeGenArgv = new ArrayList<String>();
+
+ codeGenArgv.add("--table");
+ codeGenArgv.add(getTableName(dt));
+ codeGenArgv.add("--connect");
+ codeGenArgv.add(MSSQLTestUtils.getDBConnectString());
+ codeGenArgv.add("--fields-terminated-by");
+ codeGenArgv.add("\\t");
+ codeGenArgv.add("--lines-terminated-by");
+ codeGenArgv.add("\\n");
+
+ return codeGenArgv.toArray(new String[0]);
+ }
+
+ protected String[] getArgv(DATATYPES dt, String... additionalArgv) {
+ ArrayList<String> args = new ArrayList<String>();
+
+ // Any additional Hadoop flags (-D foo=bar) are prepended.
+ if (null != additionalArgv) {
+ boolean prevIsFlag = false;
+ for (String arg : additionalArgv) {
+ if (arg.equals("-D")) {
+ args.add(arg);
+ prevIsFlag = true;
+ } else if (prevIsFlag) {
+ args.add(arg);
+ prevIsFlag = false;
+ }
+ }
+ }
+
+ // The sqoop-specific additional args are then added.
+ if (null != additionalArgv) {
+ boolean prevIsFlag = false;
+ for (String arg : additionalArgv) {
+ if (arg.equals("-D")) {
+ prevIsFlag = true;
+ continue;
+ } else if (prevIsFlag) {
+ prevIsFlag = false;
+ continue;
+ } else {
+ // normal argument.
+ args.add(arg);
+ }
+ }
+ }
+
+ args.add("--table");
+ args.add(getTableName(dt));
+ args.add("--export-dir");
+ args.add(getTablePath(dt).toString());
+ args.add("--connect");
+ args.add(MSSQLTestUtils.getDBConnectString());
+ args.add("--fields-terminated-by");
+ args.add("\\t");
+ args.add("--lines-terminated-by");
+ args.add("\\n");
+ args.add("-m");
+ args.add("1");
+
+ LOG.debug("args:");
+ for (String a : args) {
+ LOG.debug(" " + a);
+ }
+
+ return args.toArray(new String[0]);
+ }
+
+ public String getOutputFileName() {
+ return "ManagerCompatExportSeq.txt";
+ }
+
+}
[3/4] sqoop git commit: SQOOP-3174: Add SQLServer manual tests to 3rd
party test suite
Posted by an...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java
deleted file mode 100644
index 9c20bca..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.manager.sqlserver;
-
-import java.io.BufferedReader;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES;
-import com.cloudera.sqoop.Sqoop;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.orm.CompilationManager;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.tool.ImportTool;
-import com.cloudera.sqoop.util.ClassLoaderStack;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * Test to import delimited file from SQL Server.
- *
- * This uses JDBC to import data from an SQLServer database to HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerDatatypeImportDelimitedFileManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerDatatypeImportDelimitedFileManualTest
- extends SQLServerDatatypeImportSequenceFileManualTest {
-
-/**
- * Create the argv to pass to Sqoop.
- *
- * @param includeHadoopFlags
- * if true, then include -D various.settings=values
- * @param colNames
- * the columns to import. If null, all columns are used.
- * @param conf
- * a Configuration specifying additional properties to use when
- * determining the arguments.
- * @return the argv as an array of strings.
-*/
- protected String[] getArgv(boolean includeHadoopFlags, String[] colNames,
- Configuration conf) {
- if (null == colNames) {
- colNames = getColNames();
- }
-
- String splitByCol = colNames[0];
- String columnsString = "";
- for (String col : colNames) {
- columnsString += col + ",";
- }
-
- ArrayList<String> args = new ArrayList<String>();
-
- if (includeHadoopFlags) {
- CommonArgs.addHadoopFlags(args);
- }
-
- args.add("--table");
- args.add(getTableName());
- args.add("--columns");
- args.add(columnsString);
- args.add("--split-by");
- args.add(splitByCol);
- args.add("--warehouse-dir");
- args.add(getWarehouseDir());
- args.add("--connect");
- args.add(MSSQLTestUtils.getDBConnectString());
-
- args.add("--num-mappers");
- args.add("2");
-
- args.addAll(getExtraArgs(conf));
-
- return args.toArray(new String[0]);
- }
-
-
- private void runSqoopImport(String[] importCols) {
- Configuration conf = getConf();
- SqoopOptions opts = getSqoopOptions(conf);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- opts.setUsername(username);
- opts.setPassword(password);
-
- // run the tool through the normal entry-point.
- int ret;
- try {
- Sqoop importer = new Sqoop(new ImportTool(), conf, opts);
- ret = Sqoop.runSqoop(importer, getArgv(true, importCols, conf));
- } catch (Exception e) {
- LOG.error("Got exception running Sqoop: " + e.toString());
- throw new RuntimeException(e);
- }
-
- // expect a successful return.
- assertEquals("Failure during job", 0, ret);
- }
-
- /**
- * Do a MapReduce-based import of the table and verify that the results were
- * imported as expected. (tests readFields(ResultSet) and toString())
- *
- * @param expectedVal
- * the value we injected into the table.
- * @param importCols
- * the columns to import. If null, all columns are used.
- */
- protected void verifyImport(String expectedVal, String[] importCols) {
-
- // paths to where our output file will wind up.
- Path tableDirPath = getTablePath();
-
- removeTableDir();
-
- runSqoopImport(importCols);
- Configuration conf = getConf();
-
- SqoopOptions opts = getSqoopOptions(conf);
- try {
- ImportTool importTool = new ImportTool();
- opts = importTool.parseArguments(getArgv(false, importCols, conf),
- conf, opts, true);
- } catch (Exception e) {
- LOG.error(StringUtils.stringifyException(e));
- fail(e.toString());
- }
-
- CompilationManager compileMgr = new CompilationManager(opts);
- String jarFileName = compileMgr.getJarFilename();
- ClassLoader prevClassLoader = null;
- try {
- prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
- getTableName());
-
- // Now open and check all part-files in the table path until we find
- // a non-empty one that we can verify contains the value.
-
- FileSystem fs = FileSystem.getLocal(conf);
- FileStatus[] stats = fs.listStatus(tableDirPath);
-
- if (stats == null || stats.length == 0) {
- fail("Error: no files in " + tableDirPath);
- }
-
- boolean foundRecord = false;
- for (FileStatus stat : stats) {
- if (!stat.getPath().getName().startsWith("part-")
- && !stat.getPath().getName().startsWith("data-")) {
- // This isn't a data file. Ignore it.
- continue;
- }
-
- try {
- String line;
- String fname = stat.getPath().toString();
- fname = fname.substring(5, fname.length());
-
- BufferedReader reader = new BufferedReader(
- new InputStreamReader(new FileInputStream(new File(
- fname))));
- try {
- line = reader.readLine();
- assertEquals(" expected a different string",
- expectedVal, line);
- } finally {
- IOUtils.closeStream(reader);
- }
- LOG.info("Read back from delimited file: " + line);
- foundRecord = true;
- // Add trailing '\n' to expected value since
- // SqoopRecord.toString()
- // encodes the record delim.
- if (null == expectedVal) {
- assertEquals("Error validating result from delimited file",
- "null\n", line);
- }
- } catch (EOFException eoe) {
- // EOF in a file isn't necessarily a problem. We may have
- // some
- // empty sequence files, which will throw this. Just
- // continue
- // in the loop.
- }
- }
-
- if (!foundRecord) {
- fail("Couldn't read any records from delimited file");
- }
- } catch (IOException ioe) {
- LOG.error(StringUtils.stringifyException(ioe));
- fail("IOException: " + ioe.toString());
- } finally {
- if (null != prevClassLoader) {
- ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
- }
- }
- }
-
- @Test
- public void testTime() {
- if (!supportsTime()) {
- skipped = true;
- return;
- }
-
- dataTypeTest(DATATYPES.TIME);
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testVarBinary() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testBit() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testBit2() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testBit3() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testNChar() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testChar() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testVarchar() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testNVarchar() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testBinary() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testTimestamp3() {
- }
-
- public String getResportFileName(){
- return this.getClass().toString()+".txt";
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileTest.java
new file mode 100644
index 0000000..a4d1822
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileTest.java
@@ -0,0 +1,306 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.manager.sqlserver;
+
+import java.io.BufferedReader;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES;
+import com.cloudera.sqoop.Sqoop;
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.orm.CompilationManager;
+import com.cloudera.sqoop.testutil.CommonArgs;
+import com.cloudera.sqoop.tool.ImportTool;
+import com.cloudera.sqoop.util.ClassLoaderStack;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Test to import delimited file from SQL Server.
+ *
+ * This uses JDBC to import data from an SQLServer database to HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerDatatypeImportDelimitedFileTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerDatatypeImportDelimitedFileTest
+ extends SQLServerDatatypeImportSequenceFileTest {
+
+/**
+ * Create the argv to pass to Sqoop.
+ *
+ * @param includeHadoopFlags
+ * if true, then include -D various.settings=values
+ * @param colNames
+ * the columns to import. If null, all columns are used.
+ * @param conf
+ * a Configuration specifying additional properties to use when
+ * determining the arguments.
+ * @return the argv as an array of strings.
+*/
+ protected String[] getArgv(boolean includeHadoopFlags, String[] colNames,
+ Configuration conf) {
+ if (null == colNames) {
+ colNames = getColNames();
+ }
+
+ String splitByCol = colNames[0];
+ String columnsString = "";
+ for (String col : colNames) {
+ columnsString += col + ",";
+ }
+
+ ArrayList<String> args = new ArrayList<String>();
+
+ if (includeHadoopFlags) {
+ CommonArgs.addHadoopFlags(args);
+ }
+
+ args.add("--table");
+ args.add(getTableName());
+ args.add("--columns");
+ args.add(columnsString);
+ args.add("--split-by");
+ args.add(splitByCol);
+ args.add("--warehouse-dir");
+ args.add(getWarehouseDir());
+ args.add("--connect");
+ args.add(MSSQLTestUtils.getDBConnectString());
+
+ args.add("--num-mappers");
+ args.add("2");
+
+ args.addAll(getExtraArgs(conf));
+
+ return args.toArray(new String[0]);
+ }
+
+
+ private void runSqoopImport(String[] importCols) {
+ Configuration conf = getConf();
+ SqoopOptions opts = getSqoopOptions(conf);
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ opts.setUsername(username);
+ opts.setPassword(password);
+
+ // run the tool through the normal entry-point.
+ int ret;
+ try {
+ Sqoop importer = new Sqoop(new ImportTool(), conf, opts);
+ ret = Sqoop.runSqoop(importer, getArgv(true, importCols, conf));
+ } catch (Exception e) {
+ LOG.error("Got exception running Sqoop: " + e.toString());
+ throw new RuntimeException(e);
+ }
+
+ // expect a successful return.
+ assertEquals("Failure during job", 0, ret);
+ }
+
+ /**
+ * Do a MapReduce-based import of the table and verify that the results were
+ * imported as expected. (tests readFields(ResultSet) and toString())
+ *
+ * @param expectedVal
+ * the value we injected into the table.
+ * @param importCols
+ * the columns to import. If null, all columns are used.
+ */
+ protected void verifyImport(String expectedVal, String[] importCols) {
+
+ // paths to where our output file will wind up.
+ Path tableDirPath = getTablePath();
+
+ removeTableDir();
+
+ runSqoopImport(importCols);
+ Configuration conf = getConf();
+
+ SqoopOptions opts = getSqoopOptions(conf);
+ try {
+ ImportTool importTool = new ImportTool();
+ opts = importTool.parseArguments(getArgv(false, importCols, conf),
+ conf, opts, true);
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ fail(e.toString());
+ }
+
+ CompilationManager compileMgr = new CompilationManager(opts);
+ String jarFileName = compileMgr.getJarFilename();
+ ClassLoader prevClassLoader = null;
+ try {
+ prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
+ getTableName());
+
+ // Now open and check all part-files in the table path until we find
+ // a non-empty one that we can verify contains the value.
+
+ FileSystem fs = FileSystem.getLocal(conf);
+ FileStatus[] stats = fs.listStatus(tableDirPath);
+
+ if (stats == null || stats.length == 0) {
+ fail("Error: no files in " + tableDirPath);
+ }
+
+ boolean foundRecord = false;
+ for (FileStatus stat : stats) {
+ if (!stat.getPath().getName().startsWith("part-")
+ && !stat.getPath().getName().startsWith("data-")) {
+ // This isn't a data file. Ignore it.
+ continue;
+ }
+
+ try {
+ String line;
+ String fname = stat.getPath().toString();
+ fname = fname.substring(5, fname.length());
+
+ BufferedReader reader = new BufferedReader(
+ new InputStreamReader(new FileInputStream(new File(
+ fname))));
+ try {
+ line = reader.readLine();
+ assertEquals(" expected a different string",
+ expectedVal, line);
+ } finally {
+ IOUtils.closeStream(reader);
+ }
+ LOG.info("Read back from delimited file: " + line);
+ foundRecord = true;
+ // Add trailing '\n' to expected value since
+ // SqoopRecord.toString()
+ // encodes the record delim.
+ if (null == expectedVal) {
+ assertEquals("Error validating result from delimited file",
+ "null\n", line);
+ }
+ } catch (EOFException eoe) {
+ // EOF in a file isn't necessarily a problem. We may have
+ // some
+ // empty sequence files, which will throw this. Just
+ // continue
+ // in the loop.
+ }
+ }
+
+ if (!foundRecord) {
+ fail("Couldn't read any records from delimited file");
+ }
+ } catch (IOException ioe) {
+ LOG.error(StringUtils.stringifyException(ioe));
+ fail("IOException: " + ioe.toString());
+ } finally {
+ if (null != prevClassLoader) {
+ ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
+ }
+ }
+ }
+
+ @Test
+ public void testTime() {
+ if (!supportsTime()) {
+ skipped = true;
+ return;
+ }
+
+ dataTypeTest(DATATYPES.TIME);
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testVarBinary() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testBit() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testBit2() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testBit3() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testNChar() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testChar() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testVarchar() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testNVarchar() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testBinary() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testTimestamp3() {
+ }
+
+ public String getResportFileName(){
+ return this.getClass().toString()+".txt";
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java
deleted file mode 100644
index 9cd3176..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java
+++ /dev/null
@@ -1,845 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.manager.sqlserver;
-
-import java.io.FileWriter;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.StringUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.testutil.ManagerCompatTestCase;
-import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES;
-import org.apache.sqoop.manager.sqlserver.MSSQLTestData.KEY_STRINGS;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Test importing sequence file from SQL Server.
- *
- * This uses JDBC to import data from an SQLServer database to HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerDatatypeImportSequenceFileManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerDatatypeImportSequenceFileManualTest extends
- ManagerCompatTestCase {
-
- public static final Log LOG = LogFactory.getLog(
- SQLServerDatatypeImportSequenceFileManualTest.class.getName());
- private static MSSQLTestDataFileParser tdfs;
- private static Map report;
-
- static {
- try {
-
- String testfile = null;
- testfile = System.getProperty("test.data.dir")
- + "/" + System.getProperty("ms.datatype.test.data.file.import");
- String delim = System.getProperty("ms.datatype.test.data.file.delim", ",");
- System.out.println("Using data file : " + testfile);
- LOG.info("Using data file : " + testfile);
- tdfs = new MSSQLTestDataFileParser(testfile);
- tdfs.setDelim(delim);
- tdfs.parse();
- report = new HashMap();
- } catch (Exception e) {
- LOG.error(StringUtils.stringifyException(e));
- System.out
- .println("Error with test data file, check stack trace for cause"
- + ".\nTests cannont continue.");
- System.exit(0);
- }
- }
-
- @Override
- protected String getDbFriendlyName() {
- return "MSSQL";
- }
-
- @Override
- protected Log getLogger() {
- return LOG;
- }
-
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- protected String getConnectString() {
- return MSSQLTestUtils.getDBConnectString();
- }
-
- /**
- * Drop a table if it already exists in the database.
- *
- * @param table
- * the name of the table to drop.
- * @throws SQLException
- * if something goes wrong.
- */
- protected void dropTableIfExists(String table) throws SQLException {
- Connection conn = getManager().getConnection();
- String sqlStmt = "IF OBJECT_ID('" + table
- + "') IS NOT NULL DROP TABLE " + table;
-
- PreparedStatement statement = conn.prepareStatement(sqlStmt,
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- conn.commit();
- } finally {
- statement.close();
- }
- }
-
- protected SqoopOptions getSqoopOptions(Configuration conf) {
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- SqoopOptions opts = new SqoopOptions(conf);
- opts.setUsername(username);
- opts.setPassword(password);
- return opts;
- }
-
- @Before
- public void setUp() {
- try {
- super.setUp();
- } catch (Exception e) {
- try {
- FileWriter fr = new FileWriter(getResportFileName(), true);
- String res = removeNewLines(e.getMessage());
- fr.append("Error\t" + res + "\n");
- fr.close();
- } catch (Exception e2) {
- LOG.error(StringUtils.stringifyException(e2));
- fail(e2.toString());
- }
- } catch (Error e) {
- try {
- FileWriter fr = new FileWriter(getResportFileName(), true);
-
- String res = removeNewLines(e.getMessage());
-
- fr.append("Error\t" + res + "\n");
- fr.close();
- fail(res);
- } catch (Exception e2) {
- LOG.error(StringUtils.stringifyException(e2));
- fail(e2.toString());
- }
- }
- }
-
- @After
- public void tearDown() {
- try {
- super.tearDown();
- } catch (Exception e) {
- try {
- FileWriter fr = new FileWriter(getResportFileName(), true);
- String res = removeNewLines(e.getMessage());
- fr.append("Error\t" + res + "\n");
- fr.close();
- } catch (Exception e2) {
- LOG.error(StringUtils.stringifyException(e2));
- fail(e2.toString());
- }
- } catch (Error e) {
- try {
- FileWriter fr = new FileWriter(getResportFileName(), true);
- String res = removeNewLines(e.getMessage());
- fr.append("Error\t" + res + "\n");
- fr.close();
- fail(res);
- } catch (Exception e2) {
- LOG.error(StringUtils.stringifyException(e2));
- fail(e2.toString());
- }
- }
- }
-
- protected boolean supportsBoolean() {
- return true;
- }
-
- @Test
- public void testBit() {
- if (!supportsBoolean()) {
- skipped = true;
- return;
- }
- verifyType("BIT", getTrueBoolNumericSqlInput(), getTrueBoolSeqOutput());
- }
-
- @Test
- public void testBit2() {
- if (!supportsBoolean()) {
- skipped = true;
- return;
- }
- verifyType("BIT", getFalseBoolNumericSqlInput(), getFalseBoolSeqOutput());
- }
-
- @Test
- public void testBit3() {
- if (!supportsBoolean()) {
- skipped = true;
- return;
- }
- verifyType("BIT", getFalseBoolLiteralSqlInput(), getFalseBoolSeqOutput());
- }
-
- @Test
- public void testBoolean() {
- try {
- super.testBoolean();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testBoolean2() {
- try {
- super.testBoolean2();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testBoolean3() {
- try {
- super.testBoolean3();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testDouble1() {
- try {
- super.testDouble1();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testDouble2() {
- try {
- super.testDouble2();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testClob1() {
- try {
- super.testClob1();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testBlob1() {
- try {
- super.testBlob1();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testLongVarChar() {
- try {
- super.testLongVarChar();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testTimestamp1() {
- try {
- super.testTimestamp1();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testTimestamp2() {
- try {
- super.testTimestamp2();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testTimestamp3() {
- try {
- super.testTimestamp3();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testVarBinary() {
- if (!supportsVarBinary()) {
- return;
- }
- dataTypeTest(DATATYPES.VARBINARY);
- }
-
- @Test
- public void testTime() {
- if (!supportsTime()) {
- skipped = true;
- return;
- }
- dataTypeTest(DATATYPES.TIME);
- }
-
- @Test
- public void testSmalldatetime() {
- if (!supportsTime()) {
- skipped = true;
- return;
- }
- dataTypeTest(DATATYPES.SMALLDATETIME);
- }
-
- @Test
- public void testdatetime2() {
- if (!supportsTime()) {
- skipped = true;
- return;
- }
- dataTypeTest(DATATYPES.DATETIME2);
- }
-
- @Test
- public void testdatetime() {
- if (!supportsTime()) {
- skipped = true;
- return;
- }
- dataTypeTest(DATATYPES.DATETIME);
- }
-
- @Test
- public void testdatetimeoffset() {
- if (!supportsTime()) {
- skipped = true;
- return;
- }
- dataTypeTest(DATATYPES.DATETIMEOFFSET);
- }
-
- @Test
- public void testDecimal() {
- dataTypeTest(DATATYPES.DECIMAL);
- }
-
- @Test
- public void testNumeric() {
- dataTypeTest(DATATYPES.NUMERIC);
- }
-
- @Test
- public void testNumeric1() {
- }
-
- @Test
- public void testNumeric2() {
- }
-
- @Test
- public void testDecimal1() {
- }
-
- @Test
- public void testDecimal2() {
- }
-
- @Test
- public void testBigInt() {
- dataTypeTest(DATATYPES.BIGINT);
- }
-
- @Test
- public void testBigInt1() {
- }
-
- @Test
- public void testInt() {
- dataTypeTest(DATATYPES.INT);
- }
-
- @Test
- public void testSmallInt() {
- dataTypeTest(DATATYPES.SMALLINT);
- }
-
- @Test
- public void testSmallInt1() {
- }
-
- @Test
- public void testSmallInt2() {
- }
-
- @Test
- public void testTinyint() {
- dataTypeTest(DATATYPES.TINYINT);
-
- }
-
- @Test
- public void testTinyInt1() {
- }
-
- @Test
- public void testTinyInt2() {
- }
-
- @Test
- public void testFloat() {
- dataTypeTest(DATATYPES.FLOAT);
- }
-
- @Test
- public void testReal() {
- dataTypeTest(DATATYPES.REAL);
- }
-
- @Test
- public void testDate() {
- dataTypeTest(DATATYPES.DATE);
- }
-
- @Test
- public void testMoney() {
- dataTypeTest(DATATYPES.MONEY);
- }
-
- @Test
- public void testSmallMoney() {
- dataTypeTest(DATATYPES.SMALLMONEY);
- }
-
- @Test
- public void testText() {
- dataTypeTest(DATATYPES.TEXT);
- }
-
- @Test
- public void testVarchar() {
- dataTypeTest(DATATYPES.VARCHAR);
- }
-
- @Test
- public void testChar() {
- dataTypeTest(DATATYPES.CHAR);
- }
-
- @Test
- public void testNText() {
- dataTypeTest(DATATYPES.NTEXT);
- }
-
- @Test
- public void testNChar() {
- dataTypeTest(DATATYPES.NCHAR);
- }
-
- @Test
- public void testNVarchar() {
- dataTypeTest(DATATYPES.NVARCHAR);
- }
-
- @Test
- public void testImage() {
- dataTypeTest(DATATYPES.IMAGE);
- }
-
- @Test
- public void testBinary() {
- dataTypeTest(DATATYPES.BINARY);
- }
-
- //---------------disabled tests-----
- @Test
- public void testTime1() {
- }
-
- @Test
- public void testTime2() {
- }
-
- @Test
- public void testTime3() {
- }
-
- @Test
- public void testTime4() {
- }
-
- @Test
- public void testStringCol1() {
-
- }
-
- @Test
- public void testStringCol2() {
-
- }
-
- @Test
- public void testEmptyStringCol() {
-
- }
-
- @Test
- public void testNullStringCol() {
-
- }
-
- @Test
- public void testNullInt() {
-
- }
-
- @Test
- public void testReal1() {
-
- }
-
- @Test
- public void testReal2() {
-
- }
-
- @Test
- public void testFloat1() {
-
- }
-
- @Test
- public void testFloat2() {
-
- }
-
- @Test
- public void testDate1() {
-
- }
-
- @Test
- public void testDate2() {
-
- }
-
-
- @Test
- public void testNumeric3() {
-
- }
-
- @Test
- public void testNumeric4() {
-
- }
-
- @Test
- public void testNumeric5() {
-
-
- }
-
- @Test
- public void testNumeric6() {
-
- }
-
-
-
- @Test
- public void testDecimal3() {
-
- }
-
- @Test
- public void testDecimal4() {
-
- }
-
- @Test
- public void testDecimal5() {
-
-
- }
-
- @Test
- public void testDecimal6() {
-
- }
-
-
-
- //end disabled tests----------------------------
-
- public String getTrueBoolDbOutput() {
- return "1";
- }
-
- public String getFalseBoolDbOutput() {
- return "0";
- }
-
- protected String getFalseBoolSeqOutput() {
- return "false";
- }
-
- protected String getFalseBoolLiteralSqlInput() {
- return "0";
- }
-
- protected String getFixedCharSeqOut(int len, String val) {
- return val + nSpace(len - val.length());
- }
-
- protected String getFixedCharDbOut(int len, String val) {
- return val + nSpace(len - val.length());
- }
-
- public String nSpace(int n) {
- String tmp = "";
- for (int i = 0; i < n; i++) {
- tmp += " ";
- }
-
- return tmp;
- }
-
- public String nZeros(int n) {
- String tmp = "";
- for (int i = 0; i < n; i++) {
- tmp += "0";
- }
-
- return tmp;
- }
-
- public void dataTypeTest(DATATYPES datatype) {
- int exceptionCount = 0;
-
- List testdata = tdfs.getTestdata(datatype);
-
- for (Iterator<MSSQLTestData> itr = testdata.iterator(); itr.hasNext();) {
- MSSQLTestData current = itr.next();
- System.out.println("Testing with : \n" + current);
-
- try {
- if (datatype == DATATYPES.DECIMAL
- || datatype == DATATYPES.NUMERIC) {
-
- verifyType(current.getDatatype() + "("
- + current.getData(KEY_STRINGS.SCALE) + ","
- + current.getData(KEY_STRINGS.PREC) + ")", current
- .getData(KEY_STRINGS.TO_INSERT), current
- .getData(KEY_STRINGS.HDFS_READBACK));
-
- } else if (datatype == DATATYPES.TIME
- || datatype == DATATYPES.SMALLDATETIME
- || datatype == DATATYPES.DATETIME2
- || datatype == DATATYPES.DATETIME
- || datatype == DATATYPES.DATETIMEOFFSET
- || datatype == DATATYPES.TEXT
- || datatype == DATATYPES.NTEXT
- || datatype == DATATYPES.DATE) {
- verifyType(current.getDatatype(), "'"
- + current.getData(KEY_STRINGS.TO_INSERT) + "'", current
- .getData(KEY_STRINGS.HDFS_READBACK));
- } else if (datatype == DATATYPES.VARBINARY) {
- verifyType(
- current.getDatatype() + "("
- + current.getData(KEY_STRINGS.SCALE) + ")",
- "cast('" + current.getData(KEY_STRINGS.TO_INSERT)
- + "' as varbinary("
- + current.getData(KEY_STRINGS.SCALE) + "))",
- current.getData(KEY_STRINGS.HDFS_READBACK));
- } else if (datatype == DATATYPES.BINARY) {
- verifyType(
- current.getDatatype() + "("
- + current.getData(KEY_STRINGS.SCALE) + ")",
- "cast('" + current.getData(KEY_STRINGS.TO_INSERT)
- + "' as binary("
- + current.getData(KEY_STRINGS.SCALE) + "))",
- current.getData(KEY_STRINGS.HDFS_READBACK));
- } else if (datatype == DATATYPES.NCHAR
- || datatype == DATATYPES.VARCHAR
- || datatype == DATATYPES.CHAR
- || datatype == DATATYPES.NVARCHAR) {
- System.out.println("------>"
- + current.getData(KEY_STRINGS.DB_READBACK)
- + "<----");
- verifyType(current.getDatatype() + "("
- + current.getData(KEY_STRINGS.SCALE) + ")", "'"
- + current.getData(KEY_STRINGS.TO_INSERT) + "'",
- current.getData(KEY_STRINGS.HDFS_READBACK));
- } else if (datatype == DATATYPES.IMAGE) {
- verifyType(current.getDatatype(), "cast('"
- + current.getData(KEY_STRINGS.TO_INSERT)
- + "' as image )",
- current.getData(KEY_STRINGS.HDFS_READBACK));
- } else {
- verifyType(current.getDatatype(), current
- .getData(KEY_STRINGS.TO_INSERT), current
- .getData(KEY_STRINGS.HDFS_READBACK));
- }
-
- addToReport(current, null);
-
- } catch (AssertionError ae) {
- if (current.getData(KEY_STRINGS.NEG_POS_FLAG).equals("NEG")) {
- System.out.println("failure was expected, PASS");
- addToReport(current, null);
- } else {
- System.out
- .println("----------------------------------------------------------"
- + "-");
- System.out.println("Failure for following Test Data :\n"
- + current.toString());
- System.out
- .println("----------------------------------------------------------"
- + "-");
- System.out.println("Exception details : \n");
- System.out.println(ae.getMessage());
- System.out
- .println("----------------------------------------------------------"
- + "-");
- addToReport(current, ae);
- exceptionCount++;
- }
- } catch (Exception e) {
- addToReport(current, e);
- exceptionCount++;
- }
- }
-
- if (exceptionCount > 0) {
- System.out.println("There were failures for :"
- + datatype.toString());
- System.out.println("Failed for " + exceptionCount
- + " test data samples\n");
- System.out.println("Sroll up for detailed errors");
- System.out
- .println("-----------------------------------------------------------");
- throw new AssertionError("Failed for " + exceptionCount
- + " test data sample");
- }
- }
-
- public synchronized void addToReport(MSSQLTestData td, Object result) {
- System.out.println("called");
- try {
- FileWriter fr = new FileWriter(getResportFileName(), true);
- String offset = td.getData(KEY_STRINGS.OFFSET);
- String res = "_";
- if (result == null) {
- res = "Success";
- } else {
- try {
- res = "FAILED "
- + removeNewLines(((AssertionError) result)
- .getMessage());
- } catch (Exception ae) {
- if (result instanceof Exception
- && ((Exception) result) != null) {
- res = "FAILED "
- + removeNewLines(((Exception) result)
- .getMessage());
- } else {
- res = "FAILED " + result.toString();
- }
- }
- }
-
- fr.append(offset + "\t" + res + "\n");
- fr.close();
- } catch (Exception e) {
- LOG.error(StringUtils.stringifyException(e));
- }
- }
-
- public static String removeNewLines(String str) {
- String[] tmp = str.split("\n");
- String result = "";
- for (String a : tmp) {
- result += " " + a;
- }
- return result;
- }
-
- public String getResportFileName(){
- return this.getClass().toString()+".txt";
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileTest.java
new file mode 100644
index 0000000..409c4ad
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileTest.java
@@ -0,0 +1,847 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.manager.sqlserver;
+
+import java.io.FileWriter;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.testutil.ManagerCompatTestCase;
+import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES;
+import org.apache.sqoop.manager.sqlserver.MSSQLTestData.KEY_STRINGS;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test importing sequence file from SQL Server.
+ *
+ * This uses JDBC to import data from an SQLServer database to HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerDatatypeImportSequenceFileTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerDatatypeImportSequenceFileTest extends
+ ManagerCompatTestCase {
+
+ public static final Log LOG = LogFactory.getLog(
+ SQLServerDatatypeImportSequenceFileTest.class.getName());
+ private static MSSQLTestDataFileParser tdfs;
+ private static Map report;
+
+ static {
+ try {
+
+ String testfile = null;
+ testfile = System.getProperty("test.data.dir")
+ + "/" + System.getProperty("ms.datatype.test.data.file.import");
+ String delim = System.getProperty("ms.datatype.test.data.file.delim", ",");
+ System.out.println("Using data file : " + testfile);
+ LOG.info("Using data file : " + testfile);
+ tdfs = new MSSQLTestDataFileParser(testfile);
+ tdfs.setDelim(delim);
+ tdfs.parse();
+ report = new HashMap();
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ System.out
+ .println("Error with test data file, check stack trace for cause"
+ + ".\nTests cannont continue.");
+ System.exit(0);
+ }
+ }
+
+ @Override
+ protected String getDbFriendlyName() {
+ return "MSSQL";
+ }
+
+ @Override
+ protected Log getLogger() {
+ return LOG;
+ }
+
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ protected String getConnectString() {
+ return MSSQLTestUtils.getDBConnectString();
+ }
+
+ /**
+ * Drop a table if it already exists in the database.
+ *
+ * @param table
+ * the name of the table to drop.
+ * @throws SQLException
+ * if something goes wrong.
+ */
+ protected void dropTableIfExists(String table) throws SQLException {
+ Connection conn = getManager().getConnection();
+ String sqlStmt = "IF OBJECT_ID('" + table
+ + "') IS NOT NULL DROP TABLE " + table;
+
+ PreparedStatement statement = conn.prepareStatement(sqlStmt,
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ statement.executeUpdate();
+ conn.commit();
+ } finally {
+ statement.close();
+ }
+ }
+
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ SqoopOptions opts = new SqoopOptions(conf);
+ opts.setUsername(username);
+ opts.setPassword(password);
+ return opts;
+ }
+
+ @Before
+ public void setUp() {
+ try {
+ super.setUp();
+ } catch (Exception e) {
+ try {
+ FileWriter fr = new FileWriter(getResportFileName(), true);
+ String res = removeNewLines(e.getMessage());
+ fr.append("Error\t" + res + "\n");
+ fr.close();
+ } catch (Exception e2) {
+ LOG.error(StringUtils.stringifyException(e2));
+ fail(e2.toString());
+ }
+ } catch (Error e) {
+ try {
+ FileWriter fr = new FileWriter(getResportFileName(), true);
+
+ String res = removeNewLines(e.getMessage());
+
+ fr.append("Error\t" + res + "\n");
+ fr.close();
+ fail(res);
+ } catch (Exception e2) {
+ LOG.error(StringUtils.stringifyException(e2));
+ fail(e2.toString());
+ }
+ }
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ super.tearDown();
+ } catch (Exception e) {
+ try {
+ FileWriter fr = new FileWriter(getResportFileName(), true);
+ String res = removeNewLines(e.getMessage());
+ fr.append("Error\t" + res + "\n");
+ fr.close();
+ } catch (Exception e2) {
+ LOG.error(StringUtils.stringifyException(e2));
+ fail(e2.toString());
+ }
+ } catch (Error e) {
+ try {
+ FileWriter fr = new FileWriter(getResportFileName(), true);
+ String res = removeNewLines(e.getMessage());
+ fr.append("Error\t" + res + "\n");
+ fr.close();
+ fail(res);
+ } catch (Exception e2) {
+ LOG.error(StringUtils.stringifyException(e2));
+ fail(e2.toString());
+ }
+ }
+ }
+
+ protected boolean supportsBoolean() {
+ return true;
+ }
+
+ @Test
+ public void testBit() {
+ if (!supportsBoolean()) {
+ skipped = true;
+ return;
+ }
+ verifyType("BIT", getTrueBoolNumericSqlInput(), getTrueBoolSeqOutput());
+ }
+
+ @Test
+ public void testBit2() {
+ if (!supportsBoolean()) {
+ skipped = true;
+ return;
+ }
+ verifyType("BIT", getFalseBoolNumericSqlInput(), getFalseBoolSeqOutput());
+ }
+
+ @Test
+ public void testBit3() {
+ if (!supportsBoolean()) {
+ skipped = true;
+ return;
+ }
+ verifyType("BIT", getFalseBoolLiteralSqlInput(), getFalseBoolSeqOutput());
+ }
+
+ @Test
+ public void testBoolean() {
+ try {
+ super.testBoolean();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testBoolean2() {
+ try {
+ super.testBoolean2();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testBoolean3() {
+ try {
+ super.testBoolean3();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testDouble1() {
+ try {
+ super.testDouble1();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testDouble2() {
+ try {
+ super.testDouble2();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testClob1() {
+ try {
+ super.testClob1();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testBlob1() {
+ try {
+ super.testBlob1();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testLongVarChar() {
+ try {
+ super.testLongVarChar();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testTimestamp1() {
+ try {
+ super.testTimestamp1();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testTimestamp2() {
+ try {
+ super.testTimestamp2();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testTimestamp3() {
+ try {
+ super.testTimestamp3();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testVarBinary() {
+ if (!supportsVarBinary()) {
+ return;
+ }
+ dataTypeTest(DATATYPES.VARBINARY);
+ }
+
+ @Test
+ public void testTime() {
+ if (!supportsTime()) {
+ skipped = true;
+ return;
+ }
+ dataTypeTest(DATATYPES.TIME);
+ }
+
+ @Test
+ public void testSmalldatetime() {
+ if (!supportsTime()) {
+ skipped = true;
+ return;
+ }
+ dataTypeTest(DATATYPES.SMALLDATETIME);
+ }
+
+ @Test
+ public void testdatetime2() {
+ if (!supportsTime()) {
+ skipped = true;
+ return;
+ }
+ dataTypeTest(DATATYPES.DATETIME2);
+ }
+
+ @Test
+ public void testdatetime() {
+ if (!supportsTime()) {
+ skipped = true;
+ return;
+ }
+ dataTypeTest(DATATYPES.DATETIME);
+ }
+
+ @Test
+ public void testdatetimeoffset() {
+ if (!supportsTime()) {
+ skipped = true;
+ return;
+ }
+ dataTypeTest(DATATYPES.DATETIMEOFFSET);
+ }
+
+ @Test
+ public void testDecimal() {
+ dataTypeTest(DATATYPES.DECIMAL);
+ }
+
+ @Test
+ public void testNumeric() {
+ dataTypeTest(DATATYPES.NUMERIC);
+ }
+
+ @Test
+ public void testNumeric1() {
+ }
+
+ @Test
+ public void testNumeric2() {
+ }
+
+ @Test
+ public void testDecimal1() {
+ }
+
+ @Test
+ public void testDecimal2() {
+ }
+
+ @Test
+ public void testBigInt() {
+ dataTypeTest(DATATYPES.BIGINT);
+ }
+
+ @Test
+ public void testBigInt1() {
+ }
+
+ @Test
+ public void testInt() {
+ dataTypeTest(DATATYPES.INT);
+ }
+
+ @Test
+ public void testSmallInt() {
+ dataTypeTest(DATATYPES.SMALLINT);
+ }
+
+ @Test
+ public void testSmallInt1() {
+ }
+
+ @Test
+ public void testSmallInt2() {
+ }
+
+ @Test
+ public void testTinyint() {
+ dataTypeTest(DATATYPES.TINYINT);
+
+ }
+
+ @Test
+ public void testTinyInt1() {
+ }
+
+ @Test
+ public void testTinyInt2() {
+ }
+
+ @Test
+ public void testFloat() {
+ dataTypeTest(DATATYPES.FLOAT);
+ }
+
+ @Test
+ public void testReal() {
+ dataTypeTest(DATATYPES.REAL);
+ }
+
+ @Test
+ public void testDate() {
+ dataTypeTest(DATATYPES.DATE);
+ }
+
+ @Test
+ public void testMoney() {
+ dataTypeTest(DATATYPES.MONEY);
+ }
+
+ @Test
+ public void testSmallMoney() {
+ dataTypeTest(DATATYPES.SMALLMONEY);
+ }
+
+ @Test
+ public void testText() {
+ dataTypeTest(DATATYPES.TEXT);
+ }
+
+ @Test
+ public void testVarchar() {
+ dataTypeTest(DATATYPES.VARCHAR);
+ }
+
+ @Test
+ public void testChar() {
+ dataTypeTest(DATATYPES.CHAR);
+ }
+
+ @Test
+ public void testNText() {
+ dataTypeTest(DATATYPES.NTEXT);
+ }
+
+ @Test
+ public void testNChar() {
+ dataTypeTest(DATATYPES.NCHAR);
+ }
+
+ @Test
+ public void testNVarchar() {
+ dataTypeTest(DATATYPES.NVARCHAR);
+ }
+
+ @Test
+ public void testImage() {
+ dataTypeTest(DATATYPES.IMAGE);
+ }
+
+ @Test
+ public void testBinary() {
+ dataTypeTest(DATATYPES.BINARY);
+ }
+
+ //---------------disabled tests-----
+ @Test
+ public void testTime1() {
+ }
+
+ @Test
+ public void testTime2() {
+ }
+
+ @Test
+ public void testTime3() {
+ }
+
+ @Test
+ public void testTime4() {
+ }
+
+ @Test
+ public void testStringCol1() {
+
+ }
+
+ @Test
+ public void testStringCol2() {
+
+ }
+
+ @Test
+ public void testEmptyStringCol() {
+
+ }
+
+ @Test
+ public void testNullStringCol() {
+
+ }
+
+ @Test
+ public void testNullInt() {
+
+ }
+
+ @Test
+ public void testReal1() {
+
+ }
+
+ @Test
+ public void testReal2() {
+
+ }
+
+ @Test
+ public void testFloat1() {
+
+ }
+
+ @Test
+ public void testFloat2() {
+
+ }
+
+ @Test
+ public void testDate1() {
+
+ }
+
+ @Test
+ public void testDate2() {
+
+ }
+
+
+ @Test
+ public void testNumeric3() {
+
+ }
+
+ @Test
+ public void testNumeric4() {
+
+ }
+
+ @Test
+ public void testNumeric5() {
+
+
+ }
+
+ @Test
+ public void testNumeric6() {
+
+ }
+
+
+
+ @Test
+ public void testDecimal3() {
+
+ }
+
+ @Test
+ public void testDecimal4() {
+
+ }
+
+ @Test
+ public void testDecimal5() {
+
+
+ }
+
+ @Test
+ public void testDecimal6() {
+
+ }
+
+
+
+ //end disabled tests----------------------------
+
+ public String getTrueBoolDbOutput() {
+ return "1";
+ }
+
+ public String getFalseBoolDbOutput() {
+ return "0";
+ }
+
+ protected String getFalseBoolSeqOutput() {
+ return "false";
+ }
+
+ protected String getFalseBoolLiteralSqlInput() {
+ return "0";
+ }
+
+ protected String getFixedCharSeqOut(int len, String val) {
+ return val + nSpace(len - val.length());
+ }
+
+ protected String getFixedCharDbOut(int len, String val) {
+ return val + nSpace(len - val.length());
+ }
+
+ public String nSpace(int n) {
+ String tmp = "";
+ for (int i = 0; i < n; i++) {
+ tmp += " ";
+ }
+
+ return tmp;
+ }
+
+ public String nZeros(int n) {
+ String tmp = "";
+ for (int i = 0; i < n; i++) {
+ tmp += "0";
+ }
+
+ return tmp;
+ }
+
+ public void dataTypeTest(DATATYPES datatype) {
+ int exceptionCount = 0;
+
+ List testdata = tdfs.getTestdata(datatype);
+
+ for (Iterator<MSSQLTestData> itr = testdata.iterator(); itr.hasNext();) {
+ MSSQLTestData current = itr.next();
+ System.out.println("Testing with : \n" + current);
+
+ try {
+ if (datatype == DATATYPES.DECIMAL
+ || datatype == DATATYPES.NUMERIC) {
+
+ verifyType(current.getDatatype() + "("
+ + current.getData(KEY_STRINGS.SCALE) + ","
+ + current.getData(KEY_STRINGS.PREC) + ")", current
+ .getData(KEY_STRINGS.TO_INSERT), current
+ .getData(KEY_STRINGS.HDFS_READBACK));
+
+ } else if (datatype == DATATYPES.TIME
+ || datatype == DATATYPES.SMALLDATETIME
+ || datatype == DATATYPES.DATETIME2
+ || datatype == DATATYPES.DATETIME
+ || datatype == DATATYPES.DATETIMEOFFSET
+ || datatype == DATATYPES.TEXT
+ || datatype == DATATYPES.NTEXT
+ || datatype == DATATYPES.DATE) {
+ verifyType(current.getDatatype(), "'"
+ + current.getData(KEY_STRINGS.TO_INSERT) + "'", current
+ .getData(KEY_STRINGS.HDFS_READBACK));
+ } else if (datatype == DATATYPES.VARBINARY) {
+ verifyType(
+ current.getDatatype() + "("
+ + current.getData(KEY_STRINGS.SCALE) + ")",
+ "cast('" + current.getData(KEY_STRINGS.TO_INSERT)
+ + "' as varbinary("
+ + current.getData(KEY_STRINGS.SCALE) + "))",
+ current.getData(KEY_STRINGS.HDFS_READBACK));
+ } else if (datatype == DATATYPES.BINARY) {
+ verifyType(
+ current.getDatatype() + "("
+ + current.getData(KEY_STRINGS.SCALE) + ")",
+ "cast('" + current.getData(KEY_STRINGS.TO_INSERT)
+ + "' as binary("
+ + current.getData(KEY_STRINGS.SCALE) + "))",
+ current.getData(KEY_STRINGS.HDFS_READBACK));
+ } else if (datatype == DATATYPES.NCHAR
+ || datatype == DATATYPES.VARCHAR
+ || datatype == DATATYPES.CHAR
+ || datatype == DATATYPES.NVARCHAR) {
+ System.out.println("------>"
+ + current.getData(KEY_STRINGS.DB_READBACK)
+ + "<----");
+ verifyType(current.getDatatype() + "("
+ + current.getData(KEY_STRINGS.SCALE) + ")", "'"
+ + current.getData(KEY_STRINGS.TO_INSERT) + "'",
+ current.getData(KEY_STRINGS.HDFS_READBACK));
+ } else if (datatype == DATATYPES.IMAGE) {
+ verifyType(current.getDatatype(), "cast('"
+ + current.getData(KEY_STRINGS.TO_INSERT)
+ + "' as image )",
+ current.getData(KEY_STRINGS.HDFS_READBACK));
+ } else {
+ verifyType(current.getDatatype(), current
+ .getData(KEY_STRINGS.TO_INSERT), current
+ .getData(KEY_STRINGS.HDFS_READBACK));
+ }
+
+ addToReport(current, null);
+
+ } catch (AssertionError ae) {
+ if (current.getData(KEY_STRINGS.NEG_POS_FLAG).equals("NEG")) {
+ System.out.println("failure was expected, PASS");
+ addToReport(current, null);
+ } else {
+ System.out
+ .println("----------------------------------------------------------"
+ + "-");
+ System.out.println("Failure for following Test Data :\n"
+ + current.toString());
+ System.out
+ .println("----------------------------------------------------------"
+ + "-");
+ System.out.println("Exception details : \n");
+ System.out.println(ae.getMessage());
+ System.out
+ .println("----------------------------------------------------------"
+ + "-");
+ addToReport(current, ae);
+ exceptionCount++;
+ }
+ } catch (Exception e) {
+ addToReport(current, e);
+ exceptionCount++;
+ }
+ }
+
+ if (exceptionCount > 0) {
+ System.out.println("There were failures for :"
+ + datatype.toString());
+ System.out.println("Failed for " + exceptionCount
+ + " test data samples\n");
+ System.out.println("Sroll up for detailed errors");
+ System.out
+ .println("-----------------------------------------------------------");
+ throw new AssertionError("Failed for " + exceptionCount
+ + " test data sample");
+ }
+ }
+
+ public synchronized void addToReport(MSSQLTestData td, Object result) {
+ System.out.println("called");
+ try {
+ FileWriter fr = new FileWriter(getResportFileName(), true);
+ String offset = td.getData(KEY_STRINGS.OFFSET);
+ String res = "_";
+ if (result == null) {
+ res = "Success";
+ } else {
+ try {
+ res = "FAILED "
+ + removeNewLines(((AssertionError) result)
+ .getMessage());
+ } catch (Exception ae) {
+ if (result instanceof Exception
+ && ((Exception) result) != null) {
+ res = "FAILED "
+ + removeNewLines(((Exception) result)
+ .getMessage());
+ } else {
+ res = "FAILED " + result.toString();
+ }
+ }
+ }
+
+ fr.append(offset + "\t" + res + "\n");
+ fr.close();
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ }
+ }
+
+ public static String removeNewLines(String str) {
+ String[] tmp = str.split("\n");
+ String result = "";
+ for (String a : tmp) {
+ result += " " + a;
+ }
+ return result;
+ }
+
+ public String getResportFileName(){
+ return this.getClass().toString()+".txt";
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java
deleted file mode 100644
index ab3dd08..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.manager.sqlserver;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.StringUtils;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.hive.TestHiveImport;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.tool.SqoopTool;
-import org.junit.After;
-import org.junit.Before;
-
-import static org.junit.Assert.fail;
-
-/**
- * Test import to Hive from SQL Server.
- *
- * This uses JDBC to import data from an SQLServer database to HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerHiveImportManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerHiveImportManualTest extends TestHiveImport {
-
- @Before
- public void setUp() {
- super.setUp();
- }
-
- @After
- public void tearDown() {
- try {
- dropTableIfExists(getTableName());
- } catch (SQLException sqle) {
- LOG.info("Table clean-up failed: " + sqle);
- } finally {
- super.tearDown();
- }
- }
-
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- protected String getConnectString() {
- return MSSQLTestUtils.getDBConnectString();
- }
-
- //SQL Server pads out
- @Override
- protected String[] getTypes() {
- String[] types = { "VARCHAR(32)", "INTEGER", "VARCHAR(64)" };
- return types;
- }
-
- /**
- * Drop a table if it already exists in the database.
- * @param table
- * the name of the table to drop.
- * @throws SQLException
- * if something goes wrong.
- */
- protected void dropTableIfExists(String table) throws SQLException {
- Connection conn = getManager().getConnection();
- String sqlStmt = "IF OBJECT_ID('" + table
- + "') IS NOT NULL DROP TABLE " + table;
- PreparedStatement statement = conn.prepareStatement(sqlStmt,
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- conn.commit();
- } finally {
- statement.close();
- }
- }
-
- protected SqoopOptions getSqoopOptions(Configuration conf) {
-
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- SqoopOptions opts = new SqoopOptions(conf);
- opts.setUsername(username);
- opts.setPassword(password);
-
- return opts;
-
- }
-
- SqoopOptions getSqoopOptions(String[] args, SqoopTool tool) {
- SqoopOptions opts = null;
- try {
- opts = tool.parseArguments(args, null, null, true);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- opts.setUsername(username);
- opts.setPassword(password);
-
- } catch (Exception e) {
- LOG.error(StringUtils.stringifyException(e));
- fail("Invalid options: " + e.toString());
- }
-
- return opts;
- }
-
- protected String[] getArgv(boolean includeHadoopFlags, String[] moreArgs) {
- ArrayList<String> args = new ArrayList<String>();
- System.out.println("Overridden getArgv is called..");
- if (includeHadoopFlags) {
- CommonArgs.addHadoopFlags(args);
- }
-
- if (null != moreArgs) {
- for (String arg : moreArgs) {
- args.add(arg);
- }
- }
-
- args.add("--table");
- args.add(getTableName());
- args.add("--warehouse-dir");
- args.add(getWarehouseDir());
- args.add("--connect");
- args.add(getConnectString());
- args.add("--hive-import");
- String[] colNames = getColNames();
- if (null != colNames) {
- args.add("--split-by");
- args.add(colNames[0]);
- } else {
- fail("Could not determine column names.");
- }
-
- args.add("--num-mappers");
- args.add("1");
-
- for (String a : args) {
- LOG.debug("ARG : " + a);
- }
-
- return args.toArray(new String[0]);
- }
-
- protected String[] getCodeGenArgs() {
- ArrayList<String> args = new ArrayList<String>();
-
- args.add("--table");
- args.add(getTableName());
- args.add("--connect");
- args.add(getConnectString());
- args.add("--hive-import");
-
- return args.toArray(new String[0]);
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportTest.java
new file mode 100644
index 0000000..535e599
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportTest.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.manager.sqlserver;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.hive.TestHiveImport;
+import com.cloudera.sqoop.testutil.CommonArgs;
+import com.cloudera.sqoop.tool.SqoopTool;
+import org.junit.After;
+import org.junit.Before;
+
+import static org.junit.Assert.fail;
+
+/**
+ * Test import to Hive from SQL Server.
+ *
+ * This uses JDBC to import data from an SQLServer database to HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerHiveImportTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerHiveImportTest extends TestHiveImport {
+
+ @Before
+ public void setUp() {
+ super.setUp();
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ dropTableIfExists(getTableName());
+ } catch (SQLException sqle) {
+ LOG.info("Table clean-up failed: " + sqle);
+ } finally {
+ super.tearDown();
+ }
+ }
+
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ protected String getConnectString() {
+ return MSSQLTestUtils.getDBConnectString();
+ }
+
+ //SQL Server pads out
+ @Override
+ protected String[] getTypes() {
+ String[] types = { "VARCHAR(32)", "INTEGER", "VARCHAR(64)" };
+ return types;
+ }
+
+ /**
+ * Drop a table if it already exists in the database.
+ * @param table
+ * the name of the table to drop.
+ * @throws SQLException
+ * if something goes wrong.
+ */
+ protected void dropTableIfExists(String table) throws SQLException {
+ Connection conn = getManager().getConnection();
+ String sqlStmt = "IF OBJECT_ID('" + table
+ + "') IS NOT NULL DROP TABLE " + table;
+ PreparedStatement statement = conn.prepareStatement(sqlStmt,
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ statement.executeUpdate();
+ conn.commit();
+ } finally {
+ statement.close();
+ }
+ }
+
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ SqoopOptions opts = new SqoopOptions(conf);
+ opts.setUsername(username);
+ opts.setPassword(password);
+
+ return opts;
+
+ }
+
+ SqoopOptions getSqoopOptions(String[] args, SqoopTool tool) {
+ SqoopOptions opts = null;
+ try {
+ opts = tool.parseArguments(args, null, null, true);
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ opts.setUsername(username);
+ opts.setPassword(password);
+
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ fail("Invalid options: " + e.toString());
+ }
+
+ return opts;
+ }
+
+ protected String[] getArgv(boolean includeHadoopFlags, String[] moreArgs) {
+ ArrayList<String> args = new ArrayList<String>();
+ System.out.println("Overridden getArgv is called..");
+ if (includeHadoopFlags) {
+ CommonArgs.addHadoopFlags(args);
+ }
+
+ if (null != moreArgs) {
+ for (String arg : moreArgs) {
+ args.add(arg);
+ }
+ }
+
+ args.add("--table");
+ args.add(getTableName());
+ args.add("--warehouse-dir");
+ args.add(getWarehouseDir());
+ args.add("--connect");
+ args.add(getConnectString());
+ args.add("--hive-import");
+ String[] colNames = getColNames();
+ if (null != colNames) {
+ args.add("--split-by");
+ args.add(colNames[0]);
+ } else {
+ fail("Could not determine column names.");
+ }
+
+ args.add("--num-mappers");
+ args.add("1");
+
+ for (String a : args) {
+ LOG.debug("ARG : " + a);
+ }
+
+ return args.toArray(new String[0]);
+ }
+
+ protected String[] getCodeGenArgs() {
+ ArrayList<String> args = new ArrayList<String>();
+
+ args.add("--table");
+ args.add(getTableName());
+ args.add("--connect");
+ args.add(getConnectString());
+ args.add("--hive-import");
+
+ return args.toArray(new String[0]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java
deleted file mode 100644
index 8cc5a0b..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java
+++ /dev/null
@@ -1,366 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sqoop.manager.sqlserver;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Types;
-import java.util.Map;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.StringUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import com.cloudera.sqoop.ConnFactory;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.manager.ConnManager;
-import com.cloudera.sqoop.metastore.JobData;
-import com.cloudera.sqoop.testutil.HsqldbTestServer;
-import com.cloudera.sqoop.tool.ImportTool;
-import com.cloudera.sqoop.tool.SqoopTool;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
-
-/**
- * Test methods of the generic SqlManager implementation.
- *
- * This uses JDBC to import data from an SQLServer database to HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerManagerManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerManagerManualTest {
-
- public static final Log LOG = LogFactory.getLog(
- SQLServerManagerManualTest.class.getName());
-
- /** the name of a table that doesn't exist. */
- static final String MISSING_TABLE = "MISSING_TABLE";
-
- // instance variables populated during setUp, used during tests
- private HsqldbTestServer testServer;
- private ConnManager manager;
-
- @Before
- public void setUp() {
- MSSQLTestUtils utils = new MSSQLTestUtils();
- try {
- utils.createTableFromSQL(MSSQLTestUtils.CREATE_TALBE_LINEITEM);
- utils.populateLineItem();
- } catch (SQLException e) {
- LOG.error("Setup fail with SQLException: " + StringUtils.stringifyException(e));
- fail("Setup fail with SQLException: " + e.toString());
- }
- Configuration conf = getConf();
- SqoopOptions opts = getSqoopOptions(conf);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- opts.setUsername(username);
- opts.setPassword(password);
- opts.setConnectString(getConnectString());
- ConnFactory f = new ConnFactory(conf);
- try {
- this.manager = f.getManager(new JobData(opts, new ImportTool()));
- System.out.println("Manger : " + this.manager);
- } catch (IOException ioe) {
- LOG.error("Setup fail with IOException: " + StringUtils.stringifyException(ioe));
- fail("IOException instantiating manager: "
- + StringUtils.stringifyException(ioe));
- }
- }
-
- @After
- public void tearDown() {
- try {
-
- MSSQLTestUtils utils = new MSSQLTestUtils();
- utils.dropTableIfExists("TPCH1M_LINEITEM");
- manager.close();
- } catch (SQLException sqlE) {
- LOG.error("Got SQLException: " + sqlE.toString());
- fail("Got SQLException: " + sqlE.toString());
- }
- }
-
- @Test
- public void testListColNames() {
- String[] colNames = manager.getColumnNames(getTableName());
- assertNotNull("manager returned no colname list", colNames);
- assertEquals("Table list should be length 2", 16, colNames.length);
- String[] knownFields = MSSQLTestUtils.getColumns();
- for (int i = 0; i < colNames.length; i++) {
- assertEquals(knownFields[i], colNames[i]);
- }
- }
-
- @Test
- public void testListColTypes() {
- Map<String, Integer> types = manager.getColumnTypes(getTableName());
-
- assertNotNull("manager returned no types map", types);
- assertEquals("Map should be size=16", 16, types.size());
- assertEquals(types.get("L_ORDERKEY").intValue(), Types.INTEGER);
- assertEquals(types.get("L_COMMENT").intValue(), Types.VARCHAR);
- }
-
- @Test
- public void testMissingTableColNames() {
- // SQL Server returns an empty column list which gets translated as a
- // zero length array
- // how ever also check in case it returns null, which is also correct
- String[] colNames = manager.getColumnNames(MISSING_TABLE);
- if (colNames == null) {
- assertNull("No column names should be returned for missing table",
- colNames);
- }
- int numItems = colNames.length;
- assertEquals(0, numItems);
- }
-
- @Test
- public void testMissingTableColTypes() {
- Map<String, Integer> colTypes = manager.getColumnTypes(MISSING_TABLE);
- assertNull("No column types should be returned for missing table",
- colTypes);
- }
-
- // constants related to testReadTable()
- static final int EXPECTED_NUM_ROWS = 4;
- static final int EXPECTED_COL1_SUM = 10;
- static final int EXPECTED_COL2_SUM = 14;
-
- @Test
- public void testReadTable() {
- ResultSet results = null;
- try {
- results = manager.readTable(getTableName(), MSSQLTestUtils
- .getColumns());
-
- assertNotNull("ResultSet from readTable() is null!", results);
-
- ResultSetMetaData metaData = results.getMetaData();
- assertNotNull("ResultSetMetadata is null in readTable()", metaData);
-
- // ensure that we get the correct number of columns back
- assertEquals("Number of returned columns was unexpected!", metaData
- .getColumnCount(), 16);
-
- // should get back 4 rows. They are:
- // 1 2
- // 3 4
- // 5 6
- // 7 8
- // .. so while order isn't guaranteed, we should get back 16 on the
- // left
- // and 20 on the right.
- int sumCol1 = 0, sumCol2 = 0, rowCount = 0;
- while (results.next()) {
- rowCount++;
- sumCol1 += results.getInt(1);
- sumCol2 += results.getInt(2);
- }
-
- assertEquals("Expected 4 rows back", EXPECTED_NUM_ROWS, rowCount);
- assertEquals("Expected left sum of 10", EXPECTED_COL1_SUM, sumCol1);
- assertEquals("Expected right sum of 14", EXPECTED_COL2_SUM, sumCol2);
- } catch (SQLException sqlException) {
- LOG.error(StringUtils.stringifyException(sqlException));
- fail("SQL Exception: " + sqlException.toString());
- } finally {
- if (null != results) {
- try {
- results.close();
- } catch (SQLException sqlE) {
- LOG.error(StringUtils.stringifyException(sqlE));
- fail("SQL Exception in ResultSet.close(): "
- + sqlE.toString());
- }
- }
-
- manager.release();
- }
- }
-
- @Test
- public void testReadMissingTable() {
- ResultSet results = null;
- try {
- String[] colNames = { "*" };
- results = manager.readTable(MISSING_TABLE, colNames);
- assertNull("Expected null resultset from readTable(MISSING_TABLE)",
- results);
- } catch (SQLException sqlException) {
- // we actually expect this pass.
- } finally {
- if (null != results) {
- try {
- results.close();
- } catch (SQLException sqlE) {
- fail("SQL Exception in ResultSet.close(): "
- + sqlE.toString());
- }
- }
-
- manager.release();
- }
- }
-
- @Test
- public void testgetPrimaryKeyFromMissingTable() {
- String primaryKey = manager.getPrimaryKey(MISSING_TABLE);
- assertNull("Expected null pkey for missing table", primaryKey);
- }
-
- @Test
- public void testgetPrimaryKeyFromTableWithoutKey() {
- String primaryKey = manager.getPrimaryKey(getTableName());
- assertNull("Expected null pkey for table without key", primaryKey);
- }
-
- // constants for getPrimaryKeyFromTable()
- static final String TABLE_WITH_KEY = "TABLE_WITH_KEY";
- static final String KEY_FIELD_NAME = "KEYFIELD";
-
- @Test
- public void testgetPrimaryKeyFromTable() {
- // first, create a table with a primary key
- Connection conn = null;
- try {
- conn = getManager().getConnection();
- dropTableIfExists(TABLE_WITH_KEY);
- PreparedStatement statement = conn.prepareStatement("CREATE TABLE "
- + TABLE_WITH_KEY + "(" + KEY_FIELD_NAME
- + " INT NOT NULL PRIMARY KEY, foo INT)",
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- statement.executeUpdate();
- statement.close();
-
- String primaryKey = getManager().getPrimaryKey(TABLE_WITH_KEY);
- assertEquals("Expected null pkey for table without key",
- primaryKey, KEY_FIELD_NAME);
-
- } catch (SQLException sqlException) {
- LOG.error(StringUtils.stringifyException(sqlException));
- fail("Could not create table with primary key: "
- + sqlException.toString());
- } finally {
- if (null != conn) {
- try {
- conn.close();
- } catch (SQLException sqlE) {
- LOG.warn("Got SQLException during close: "
- + sqlE.toString());
- }
- }
- }
-
- }
-
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- protected String getConnectString() {
- return MSSQLTestUtils.getDBConnectString();
- }
-
- /**
- * Drop a table if it already exists in the database.
- *
- * @param table
- * the name of the table to drop.
- * @throws SQLException
- * if something goes wrong.
- */
- protected void dropTableIfExists(String table) throws SQLException {
- Connection conn = getManager().getConnection();
- String sqlStmt = "IF OBJECT_ID('" + table
- + "') IS NOT NULL DROP TABLE " + table;
- PreparedStatement statement = conn.prepareStatement(sqlStmt,
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- conn.commit();
- } finally {
- statement.close();
- }
- }
-
- protected SqoopOptions getSqoopOptions(Configuration conf) {
- SqoopOptions opt = new SqoopOptions(conf);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- SqoopOptions opts = new SqoopOptions(conf);
- opts.setUsername(username);
- opts.setPassword(password);
-
- return opt;
- }
-
- SqoopOptions getSqoopOptions(String[] args, SqoopTool tool) {
- SqoopOptions opts = null;
- try {
- opts = tool.parseArguments(args, null, null, true);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- opts.setUsername(username);
- opts.setPassword(password);
-
- } catch (Exception e) {
- LOG.error(StringUtils.stringifyException(e));
- fail("Invalid options: " + e.toString());
- }
-
- return opts;
- }
-
- protected String getTableName() {
- return "tpch1m_lineitem";
- }
-
- protected ConnManager getManager() {
- return manager;
- }
-
- protected Configuration getConf() {
- return new Configuration();
- }
-
-}
[2/4] sqoop git commit: SQOOP-3174: Add SQLServer manual tests to 3rd
party test suite
Posted by an...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerTest.java
new file mode 100644
index 0000000..67d8f1b
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerTest.java
@@ -0,0 +1,368 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.sqlserver;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import com.cloudera.sqoop.ConnFactory;
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.manager.ConnManager;
+import com.cloudera.sqoop.metastore.JobData;
+import com.cloudera.sqoop.testutil.HsqldbTestServer;
+import com.cloudera.sqoop.tool.ImportTool;
+import com.cloudera.sqoop.tool.SqoopTool;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
+/**
+ * Test methods of the generic SqlManager implementation.
+ *
+ * This uses JDBC to import data from an SQLServer database to HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerManagerTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerManagerTest {
+
+ public static final Log LOG = LogFactory.getLog(
+ SQLServerManagerTest.class.getName());
+
+ /** the name of a table that doesn't exist. */
+ static final String MISSING_TABLE = "MISSING_TABLE";
+
+ // instance variables populated during setUp, used during tests
+ private HsqldbTestServer testServer;
+ private ConnManager manager;
+
+ @Before
+ public void setUp() {
+ MSSQLTestUtils utils = new MSSQLTestUtils();
+ try {
+ utils.createTableFromSQL(MSSQLTestUtils.CREATE_TALBE_LINEITEM);
+ utils.populateLineItem();
+ } catch (SQLException e) {
+ LOG.error("Setup fail with SQLException: " + StringUtils.stringifyException(e));
+ fail("Setup fail with SQLException: " + e.toString());
+ }
+ Configuration conf = getConf();
+ SqoopOptions opts = getSqoopOptions(conf);
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ opts.setUsername(username);
+ opts.setPassword(password);
+ opts.setConnectString(getConnectString());
+ ConnFactory f = new ConnFactory(conf);
+ try {
+ this.manager = f.getManager(new JobData(opts, new ImportTool()));
+ System.out.println("Manger : " + this.manager);
+ } catch (IOException ioe) {
+ LOG.error("Setup fail with IOException: " + StringUtils.stringifyException(ioe));
+ fail("IOException instantiating manager: "
+ + StringUtils.stringifyException(ioe));
+ }
+ }
+
+ @After
+ public void tearDown() {
+ try {
+
+ MSSQLTestUtils utils = new MSSQLTestUtils();
+ utils.dropTableIfExists("TPCH1M_LINEITEM");
+ manager.close();
+ } catch (SQLException sqlE) {
+ LOG.error("Got SQLException: " + sqlE.toString());
+ fail("Got SQLException: " + sqlE.toString());
+ }
+ }
+
+ @Test
+ public void testListColNames() {
+ String[] colNames = manager.getColumnNames(getTableName());
+ assertNotNull("manager returned no colname list", colNames);
+ assertEquals("Table list should be length 2", 16, colNames.length);
+ String[] knownFields = MSSQLTestUtils.getColumns();
+ for (int i = 0; i < colNames.length; i++) {
+ assertEquals(knownFields[i], colNames[i]);
+ }
+ }
+
+ @Test
+ public void testListColTypes() {
+ Map<String, Integer> types = manager.getColumnTypes(getTableName());
+
+ assertNotNull("manager returned no types map", types);
+ assertEquals("Map should be size=16", 16, types.size());
+ assertEquals(types.get("L_ORDERKEY").intValue(), Types.INTEGER);
+ assertEquals(types.get("L_COMMENT").intValue(), Types.VARCHAR);
+ }
+
+ @Test
+ public void testMissingTableColNames() {
+ // SQL Server returns an empty column list which gets translated as a
+ // zero length array
+ // how ever also check in case it returns null, which is also correct
+ String[] colNames = manager.getColumnNames(MISSING_TABLE);
+ if (colNames == null) {
+ assertNull("No column names should be returned for missing table",
+ colNames);
+ }
+ int numItems = colNames.length;
+ assertEquals(0, numItems);
+ }
+
+ @Test
+ public void testMissingTableColTypes() {
+ Map<String, Integer> colTypes = manager.getColumnTypes(MISSING_TABLE);
+ assertNull("No column types should be returned for missing table",
+ colTypes);
+ }
+
+ // constants related to testReadTable()
+ static final int EXPECTED_NUM_ROWS = 4;
+ static final int EXPECTED_COL1_SUM = 10;
+ static final int EXPECTED_COL2_SUM = 14;
+
+ @Test
+ public void testReadTable() {
+ ResultSet results = null;
+ try {
+ results = manager.readTable(getTableName(), MSSQLTestUtils
+ .getColumns());
+
+ assertNotNull("ResultSet from readTable() is null!", results);
+
+ ResultSetMetaData metaData = results.getMetaData();
+ assertNotNull("ResultSetMetadata is null in readTable()", metaData);
+
+ // ensure that we get the correct number of columns back
+ assertEquals("Number of returned columns was unexpected!", metaData
+ .getColumnCount(), 16);
+
+ // should get back 4 rows. They are:
+ // 1 2
+ // 3 4
+ // 5 6
+ // 7 8
+ // .. so while order isn't guaranteed, we should get back 16 on the
+ // left
+ // and 20 on the right.
+ int sumCol1 = 0, sumCol2 = 0, rowCount = 0;
+ while (results.next()) {
+ rowCount++;
+ sumCol1 += results.getInt(1);
+ sumCol2 += results.getInt(2);
+ }
+
+ assertEquals("Expected 4 rows back", EXPECTED_NUM_ROWS, rowCount);
+ assertEquals("Expected left sum of 10", EXPECTED_COL1_SUM, sumCol1);
+ assertEquals("Expected right sum of 14", EXPECTED_COL2_SUM, sumCol2);
+ } catch (SQLException sqlException) {
+ LOG.error(StringUtils.stringifyException(sqlException));
+ fail("SQL Exception: " + sqlException.toString());
+ } finally {
+ if (null != results) {
+ try {
+ results.close();
+ } catch (SQLException sqlE) {
+ LOG.error(StringUtils.stringifyException(sqlE));
+ fail("SQL Exception in ResultSet.close(): "
+ + sqlE.toString());
+ }
+ }
+
+ manager.release();
+ }
+ }
+
+ @Test
+ public void testReadMissingTable() {
+ ResultSet results = null;
+ try {
+ String[] colNames = { "*" };
+ results = manager.readTable(MISSING_TABLE, colNames);
+ assertNull("Expected null resultset from readTable(MISSING_TABLE)",
+ results);
+ } catch (SQLException sqlException) {
+ // we actually expect this pass.
+ } finally {
+ if (null != results) {
+ try {
+ results.close();
+ } catch (SQLException sqlE) {
+ fail("SQL Exception in ResultSet.close(): "
+ + sqlE.toString());
+ }
+ }
+
+ manager.release();
+ }
+ }
+
+ @Test
+ public void testgetPrimaryKeyFromMissingTable() {
+ String primaryKey = manager.getPrimaryKey(MISSING_TABLE);
+ assertNull("Expected null pkey for missing table", primaryKey);
+ }
+
+ @Test
+ public void testgetPrimaryKeyFromTableWithoutKey() {
+ String primaryKey = manager.getPrimaryKey(getTableName());
+ assertNull("Expected null pkey for table without key", primaryKey);
+ }
+
+ // constants for getPrimaryKeyFromTable()
+ static final String TABLE_WITH_KEY = "TABLE_WITH_KEY";
+ static final String KEY_FIELD_NAME = "KEYFIELD";
+
+ @Test
+ public void testgetPrimaryKeyFromTable() {
+ // first, create a table with a primary key
+ Connection conn = null;
+ try {
+ conn = getManager().getConnection();
+ dropTableIfExists(TABLE_WITH_KEY);
+ PreparedStatement statement = conn.prepareStatement("CREATE TABLE "
+ + TABLE_WITH_KEY + "(" + KEY_FIELD_NAME
+ + " INT NOT NULL PRIMARY KEY, foo INT)",
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ statement.executeUpdate();
+ statement.close();
+
+ String primaryKey = getManager().getPrimaryKey(TABLE_WITH_KEY);
+ assertEquals("Expected null pkey for table without key",
+ primaryKey, KEY_FIELD_NAME);
+
+ } catch (SQLException sqlException) {
+ LOG.error(StringUtils.stringifyException(sqlException));
+ fail("Could not create table with primary key: "
+ + sqlException.toString());
+ } finally {
+ if (null != conn) {
+ try {
+ conn.close();
+ } catch (SQLException sqlE) {
+ LOG.warn("Got SQLException during close: "
+ + sqlE.toString());
+ }
+ }
+ }
+
+ }
+
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ protected String getConnectString() {
+ return MSSQLTestUtils.getDBConnectString();
+ }
+
+ /**
+ * Drop a table if it already exists in the database.
+ *
+ * @param table
+ * the name of the table to drop.
+ * @throws SQLException
+ * if something goes wrong.
+ */
+ protected void dropTableIfExists(String table) throws SQLException {
+ Connection conn = getManager().getConnection();
+ String sqlStmt = "IF OBJECT_ID('" + table
+ + "') IS NOT NULL DROP TABLE " + table;
+ PreparedStatement statement = conn.prepareStatement(sqlStmt,
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ statement.executeUpdate();
+ conn.commit();
+ } finally {
+ statement.close();
+ }
+ }
+
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ SqoopOptions opt = new SqoopOptions(conf);
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ SqoopOptions opts = new SqoopOptions(conf);
+ opts.setUsername(username);
+ opts.setPassword(password);
+
+ return opt;
+ }
+
+ SqoopOptions getSqoopOptions(String[] args, SqoopTool tool) {
+ SqoopOptions opts = null;
+ try {
+ opts = tool.parseArguments(args, null, null, true);
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ opts.setUsername(username);
+ opts.setPassword(password);
+
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ fail("Invalid options: " + e.toString());
+ }
+
+ return opts;
+ }
+
+ protected String getTableName() {
+ return "tpch1m_lineitem";
+ }
+
+ protected ConnManager getManager() {
+ return manager;
+ }
+
+ protected Configuration getConf() {
+ return new Configuration();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiColsManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiColsManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiColsManualTest.java
deleted file mode 100644
index 51d5f75..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiColsManualTest.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.manager.sqlserver;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import org.apache.hadoop.conf.Configuration;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.TestMultiCols;
-import org.junit.After;
-import org.junit.Test;
-
-/**
- * Test multiple columns in SQL Server.
- *
- * This uses JDBC to import data from an SQLServer database to HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerMultiColsManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerMultiColsManualTest extends TestMultiCols {
-
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- protected String getConnectString() {
- return MSSQLTestUtils.getDBConnectString();
- }
-
- /**
- * Drop a table if it already exists in the database.
- *
- * @param table
- * the name of the table to drop.
- * @throws SQLException
- * if something goes wrong.
- */
- protected void dropTableIfExists(String table) throws SQLException {
- Connection conn = getManager().getConnection();
- String sqlStmt = "IF OBJECT_ID('" + table
- + "') IS NOT NULL DROP TABLE " + table;
- PreparedStatement statement = conn.prepareStatement(sqlStmt,
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- conn.commit();
- } finally {
- statement.close();
- }
- }
-
- protected SqoopOptions getSqoopOptions(Configuration conf) {
-
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- SqoopOptions opts = new SqoopOptions(conf);
- opts.setUsername(username);
- opts.setPassword(password);
-
- return opts;
-
- }
-
- @After
- public void tearDown() {
- try {
- dropTableIfExists(getTableName());
- } catch (SQLException sqle) {
- LOG.info("Table clean-up failed: " + sqle);
- } finally {
- super.tearDown();
- }
- }
-
- @Test
- public void testMixed4() {
- // Overridden to bypass test case invalid for MSSQL server
- }
-
- @Test
- public void testMixed5() {
- // Overridden to bypass test case invalid for MSSQL server
- }
-
- @Test
- public void testMixed6() {
- // Overridden to bypass test case invalid for MSSQL server
- }
-
- @Test
- public void testSkipFirstCol() {
- // Overridden to bypass test case invalid for MSSQL server
- }
-
- @Test
- public void testSkipSecondCol() {
- // Overridden to bypass test case invalid for MSSQL server
- }
-
- @Test
- public void testSkipThirdCol() {
- // Overridden to bypass test case invalid for MSSQL server
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiColsTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiColsTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiColsTest.java
new file mode 100644
index 0000000..d48de99
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiColsTest.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.manager.sqlserver;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import org.apache.hadoop.conf.Configuration;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.TestMultiCols;
+import org.junit.After;
+import org.junit.Test;
+
+/**
+ * Test multiple columns in SQL Server.
+ *
+ * This uses JDBC to import data from an SQLServer database to HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerMultiColsTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerMultiColsTest extends TestMultiCols {
+
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ protected String getConnectString() {
+ return MSSQLTestUtils.getDBConnectString();
+ }
+
+ /**
+ * Drop a table if it already exists in the database.
+ *
+ * @param table
+ * the name of the table to drop.
+ * @throws SQLException
+ * if something goes wrong.
+ */
+ protected void dropTableIfExists(String table) throws SQLException {
+ Connection conn = getManager().getConnection();
+ String sqlStmt = "IF OBJECT_ID('" + table
+ + "') IS NOT NULL DROP TABLE " + table;
+ PreparedStatement statement = conn.prepareStatement(sqlStmt,
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ statement.executeUpdate();
+ conn.commit();
+ } finally {
+ statement.close();
+ }
+ }
+
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ SqoopOptions opts = new SqoopOptions(conf);
+ opts.setUsername(username);
+ opts.setPassword(password);
+
+ return opts;
+
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ dropTableIfExists(getTableName());
+ } catch (SQLException sqle) {
+ LOG.info("Table clean-up failed: " + sqle);
+ } finally {
+ super.tearDown();
+ }
+ }
+
+ @Test
+ public void testMixed4() {
+ // Overridden to bypass test case invalid for MSSQL server
+ }
+
+ @Test
+ public void testMixed5() {
+ // Overridden to bypass test case invalid for MSSQL server
+ }
+
+ @Test
+ public void testMixed6() {
+ // Overridden to bypass test case invalid for MSSQL server
+ }
+
+ @Test
+ public void testSkipFirstCol() {
+ // Overridden to bypass test case invalid for MSSQL server
+ }
+
+ @Test
+ public void testSkipSecondCol() {
+ // Overridden to bypass test case invalid for MSSQL server
+ }
+
+ @Test
+ public void testSkipThirdCol() {
+ // Overridden to bypass test case invalid for MSSQL server
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiMapsManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiMapsManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiMapsManualTest.java
deleted file mode 100644
index fc9e20d..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiMapsManualTest.java
+++ /dev/null
@@ -1,320 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sqoop.manager.sqlserver;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.mapred.Utils;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
-import com.cloudera.sqoop.orm.CompilationManager;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
-import com.cloudera.sqoop.testutil.SeqFileReader;
-import com.cloudera.sqoop.tool.ImportTool;
-import com.cloudera.sqoop.tool.SqoopTool;
-import com.cloudera.sqoop.util.ClassLoaderStack;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * Test multiple mapper splits in SQL Server.
- *
- * This uses JDBC to import data from an SQLServer database to HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerMultiMapsManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerMultiMapsManualTest extends ImportJobTestCase {
-
- @Before
- public void setUp() {
- super.setUp();
- MSSQLTestUtils utils = new MSSQLTestUtils();
- try {
- utils.createTableFromSQL(MSSQLTestUtils.CREATE_TALBE_LINEITEM);
- utils.populateLineItem();
- } catch (SQLException e) {
- LOG.error("Setup fail with SQLException: " + StringUtils.stringifyException(e));
- fail("Setup fail with SQLException: " + e.toString());
- }
-
- }
-
- @After
- public void tearDown() {
- super.tearDown();
- MSSQLTestUtils utils = new MSSQLTestUtils();
- try {
- utils.dropTableIfExists("TPCH1M_LINEITEM");
- } catch (SQLException e) {
- LOG.error("TeatDown fail with SQLException: " + StringUtils.stringifyException(e));
- fail("TearDown fail with SQLException: " + e.toString());
- }
- }
-
- /**
- * Create the argv to pass to Sqoop.
- *
- * @return the argv as an array of strings.
- */
- protected String[] getArgv(boolean includeHadoopFlags, String[] colNames,
- String splitByCol) {
- String columnsString = "";
- for (String col : colNames) {
- columnsString += col + ",";
- }
-
- ArrayList<String> args = new ArrayList<String>();
-
- if (includeHadoopFlags) {
- CommonArgs.addHadoopFlags(args);
- }
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
-
- args.add("--table");
- args.add(getTableName());
- args.add("--columns");
- args.add(columnsString);
- args.add("--split-by");
- args.add(splitByCol);
- args.add("--warehouse-dir");
- args.add(getWarehouseDir());
- args.add("--connect");
- args.add(getConnectString());
- args.add("--username");
- args.add(username);
- args.add("--password");
- args.add(password);
- args.add("--as-sequencefile");
- args.add("--num-mappers");
- args.add("2");
-
- return args.toArray(new String[0]);
- }
-
- // this test just uses the two int table.
-
- /** @return a list of Path objects for each data file */
- protected List<Path> getDataFilePaths() throws IOException {
- List<Path> paths = new ArrayList<Path>();
- Configuration conf = new Configuration();
- conf.set("fs.default.name", "file:///");
- FileSystem fs = FileSystem.get(conf);
-
- FileStatus[] stats = fs.listStatus(getTablePath(),
- new Utils.OutputFileUtils.OutputFilesFilter());
-
- for (FileStatus stat : stats) {
- paths.add(stat.getPath());
- }
-
- return paths;
- }
-
- /**
- * Given a comma-delimited list of integers, grab and parse the first int.
- *
- * @param str
- * a comma-delimited list of values, the first of which is an
- * int.
- * @return the first field in the string, cast to int
- */
- private int getFirstInt(String str) {
- String[] parts = str.split(",");
- return Integer.parseInt(parts[0]);
- }
-
- public void runMultiMapTest(String splitByCol, int expectedSum)
- throws IOException {
-
- String[] columns = MSSQLTestUtils.getColumns();
- ClassLoader prevClassLoader = null;
- SequenceFile.Reader reader = null;
-
- String[] argv = getArgv(true, columns, splitByCol);
- runImport(argv);
- try {
- ImportTool importTool = new ImportTool();
- SqoopOptions opts = importTool.parseArguments(getArgv(false,
- columns, splitByCol), null, null, true);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- opts.setUsername(username);
- opts.setPassword(password);
-
- CompilationManager compileMgr = new CompilationManager(opts);
- String jarFileName = compileMgr.getJarFilename();
-
- prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
- getTableName());
-
- List<Path> paths = getDataFilePaths();
- Configuration conf = new Configuration();
- int curSum = 0;
-
- // We expect multiple files. We need to open all the files and sum
- // up the
- // first column across all of them.
- for (Path p : paths) {
- reader = SeqFileReader.getSeqFileReader(p.toString());
-
- // here we can actually instantiate (k, v) pairs.
- Object key = ReflectionUtils.newInstance(reader.getKeyClass(),
- conf);
- Object val = ReflectionUtils.newInstance(
- reader.getValueClass(), conf);
-
- // We know that these values are two ints separated by a ','
- // character. Since this is all dynamic, though, we don't want
- // to
- // actually link against the class and use its methods. So we
- // just
- // parse this back into int fields manually. Sum them up and
- // ensure
- // that we get the expected total for the first column, to
- // verify that
- // we got all the results from the db into the file.
-
- // now sum up everything in the file.
- while (reader.next(key) != null) {
- reader.getCurrentValue(val);
- curSum += getFirstInt(val.toString());
- }
-
- IOUtils.closeStream(reader);
- reader = null;
- }
-
- assertEquals("Total sum of first db column mismatch", expectedSum,
- curSum);
- } catch (InvalidOptionsException ioe) {
- LOG.error(StringUtils.stringifyException(ioe));
- fail(ioe.toString());
- } catch (ParseException pe) {
- LOG.error(StringUtils.stringifyException(pe));
- fail(pe.toString());
- } finally {
- IOUtils.closeStream(reader);
-
- if (null != prevClassLoader) {
- ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
- }
- }
- }
-
- @Test
- public void testSplitByFirstCol() throws IOException {
- runMultiMapTest("L_ORDERKEY", 10);
- }
-
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- protected String getConnectString() {
- return MSSQLTestUtils.getDBConnectString();
- }
-
- /**
- * Drop a table if it already exists in the database.
- *
- * @param table
- * the name of the table to drop.
- * @throws SQLException
- * if something goes wrong.
- */
- protected void dropTableIfExists(String table) throws SQLException {
- Connection conn = getManager().getConnection();
- String sqlStmt = "IF OBJECT_ID('" + table
- + "') IS NOT NULL DROP TABLE " + table;
- PreparedStatement statement = conn.prepareStatement(sqlStmt,
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- conn.commit();
- } finally {
- statement.close();
- }
- }
-
- protected SqoopOptions getSqoopOptions(Configuration conf) {
- SqoopOptions opt = new SqoopOptions(conf);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- SqoopOptions opts = new SqoopOptions(conf);
- opts.setUsername(username);
- opts.setPassword(password);
-
- return opt;
- }
-
- SqoopOptions getSqoopOptions(String[] args, SqoopTool tool) {
- SqoopOptions opts = null;
- try {
- opts = tool.parseArguments(args, null, null, true);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- opts.setUsername(username);
- opts.setPassword(password);
-
- } catch (Exception e) {
- LOG.error(StringUtils.stringifyException(e));
- fail("Invalid options: " + e.toString());
- }
-
- return opts;
- }
-
- protected String getTableName() {
- return "tpch1m_lineitem";
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiMapsTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiMapsTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiMapsTest.java
new file mode 100644
index 0000000..be42da3
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerMultiMapsTest.java
@@ -0,0 +1,322 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.sqlserver;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.mapred.Utils;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
+import com.cloudera.sqoop.orm.CompilationManager;
+import com.cloudera.sqoop.testutil.CommonArgs;
+import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import com.cloudera.sqoop.testutil.SeqFileReader;
+import com.cloudera.sqoop.tool.ImportTool;
+import com.cloudera.sqoop.tool.SqoopTool;
+import com.cloudera.sqoop.util.ClassLoaderStack;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Test multiple mapper splits in SQL Server.
+ *
+ * This uses JDBC to import data from an SQLServer database to HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerMultiMapsTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerMultiMapsTest extends ImportJobTestCase {
+
+ @Before
+ public void setUp() {
+ super.setUp();
+ MSSQLTestUtils utils = new MSSQLTestUtils();
+ try {
+ utils.createTableFromSQL(MSSQLTestUtils.CREATE_TALBE_LINEITEM);
+ utils.populateLineItem();
+ } catch (SQLException e) {
+ LOG.error("Setup fail with SQLException: " + StringUtils.stringifyException(e));
+ fail("Setup fail with SQLException: " + e.toString());
+ }
+
+ }
+
+ @After
+ public void tearDown() {
+ super.tearDown();
+ MSSQLTestUtils utils = new MSSQLTestUtils();
+ try {
+ utils.dropTableIfExists("TPCH1M_LINEITEM");
+ } catch (SQLException e) {
+ LOG.error("TeatDown fail with SQLException: " + StringUtils.stringifyException(e));
+ fail("TearDown fail with SQLException: " + e.toString());
+ }
+ }
+
+ /**
+ * Create the argv to pass to Sqoop.
+ *
+ * @return the argv as an array of strings.
+ */
+ protected String[] getArgv(boolean includeHadoopFlags, String[] colNames,
+ String splitByCol) {
+ String columnsString = "";
+ for (String col : colNames) {
+ columnsString += col + ",";
+ }
+
+ ArrayList<String> args = new ArrayList<String>();
+
+ if (includeHadoopFlags) {
+ CommonArgs.addHadoopFlags(args);
+ }
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+
+ args.add("--table");
+ args.add(getTableName());
+ args.add("--columns");
+ args.add(columnsString);
+ args.add("--split-by");
+ args.add(splitByCol);
+ args.add("--warehouse-dir");
+ args.add(getWarehouseDir());
+ args.add("--connect");
+ args.add(getConnectString());
+ args.add("--username");
+ args.add(username);
+ args.add("--password");
+ args.add(password);
+ args.add("--as-sequencefile");
+ args.add("--num-mappers");
+ args.add("2");
+
+ return args.toArray(new String[0]);
+ }
+
+ // this test just uses the two int table.
+
+ /** @return a list of Path objects for each data file */
+ protected List<Path> getDataFilePaths() throws IOException {
+ List<Path> paths = new ArrayList<Path>();
+ Configuration conf = new Configuration();
+ conf.set("fs.default.name", "file:///");
+ FileSystem fs = FileSystem.get(conf);
+
+ FileStatus[] stats = fs.listStatus(getTablePath(),
+ new Utils.OutputFileUtils.OutputFilesFilter());
+
+ for (FileStatus stat : stats) {
+ paths.add(stat.getPath());
+ }
+
+ return paths;
+ }
+
+ /**
+ * Given a comma-delimited list of integers, grab and parse the first int.
+ *
+ * @param str
+ * a comma-delimited list of values, the first of which is an
+ * int.
+ * @return the first field in the string, cast to int
+ */
+ private int getFirstInt(String str) {
+ String[] parts = str.split(",");
+ return Integer.parseInt(parts[0]);
+ }
+
+ public void runMultiMapTest(String splitByCol, int expectedSum)
+ throws IOException {
+
+ String[] columns = MSSQLTestUtils.getColumns();
+ ClassLoader prevClassLoader = null;
+ SequenceFile.Reader reader = null;
+
+ String[] argv = getArgv(true, columns, splitByCol);
+ runImport(argv);
+ try {
+ ImportTool importTool = new ImportTool();
+ SqoopOptions opts = importTool.parseArguments(getArgv(false,
+ columns, splitByCol), null, null, true);
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ opts.setUsername(username);
+ opts.setPassword(password);
+
+ CompilationManager compileMgr = new CompilationManager(opts);
+ String jarFileName = compileMgr.getJarFilename();
+
+ prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
+ getTableName());
+
+ List<Path> paths = getDataFilePaths();
+ Configuration conf = new Configuration();
+ int curSum = 0;
+
+ // We expect multiple files. We need to open all the files and sum
+ // up the
+ // first column across all of them.
+ for (Path p : paths) {
+ reader = SeqFileReader.getSeqFileReader(p.toString());
+
+ // here we can actually instantiate (k, v) pairs.
+ Object key = ReflectionUtils.newInstance(reader.getKeyClass(),
+ conf);
+ Object val = ReflectionUtils.newInstance(
+ reader.getValueClass(), conf);
+
+ // We know that these values are two ints separated by a ','
+ // character. Since this is all dynamic, though, we don't want
+ // to
+ // actually link against the class and use its methods. So we
+ // just
+ // parse this back into int fields manually. Sum them up and
+ // ensure
+ // that we get the expected total for the first column, to
+ // verify that
+ // we got all the results from the db into the file.
+
+ // now sum up everything in the file.
+ while (reader.next(key) != null) {
+ reader.getCurrentValue(val);
+ curSum += getFirstInt(val.toString());
+ }
+
+ IOUtils.closeStream(reader);
+ reader = null;
+ }
+
+ assertEquals("Total sum of first db column mismatch", expectedSum,
+ curSum);
+ } catch (InvalidOptionsException ioe) {
+ LOG.error(StringUtils.stringifyException(ioe));
+ fail(ioe.toString());
+ } catch (ParseException pe) {
+ LOG.error(StringUtils.stringifyException(pe));
+ fail(pe.toString());
+ } finally {
+ IOUtils.closeStream(reader);
+
+ if (null != prevClassLoader) {
+ ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
+ }
+ }
+ }
+
+ @Test
+ public void testSplitByFirstCol() throws IOException {
+ runMultiMapTest("L_ORDERKEY", 10);
+ }
+
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ protected String getConnectString() {
+ return MSSQLTestUtils.getDBConnectString();
+ }
+
+ /**
+ * Drop a table if it already exists in the database.
+ *
+ * @param table
+ * the name of the table to drop.
+ * @throws SQLException
+ * if something goes wrong.
+ */
+ protected void dropTableIfExists(String table) throws SQLException {
+ Connection conn = getManager().getConnection();
+ String sqlStmt = "IF OBJECT_ID('" + table
+ + "') IS NOT NULL DROP TABLE " + table;
+ PreparedStatement statement = conn.prepareStatement(sqlStmt,
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ statement.executeUpdate();
+ conn.commit();
+ } finally {
+ statement.close();
+ }
+ }
+
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ SqoopOptions opt = new SqoopOptions(conf);
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ SqoopOptions opts = new SqoopOptions(conf);
+ opts.setUsername(username);
+ opts.setPassword(password);
+
+ return opt;
+ }
+
+ SqoopOptions getSqoopOptions(String[] args, SqoopTool tool) {
+ SqoopOptions opts = null;
+ try {
+ opts = tool.parseArguments(args, null, null, true);
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ opts.setUsername(username);
+ opts.setPassword(password);
+
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ fail("Invalid options: " + e.toString());
+ }
+
+ return opts;
+ }
+
+ protected String getTableName() {
+ return "tpch1m_lineitem";
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsManualTest.java
deleted file mode 100644
index b28c165..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsManualTest.java
+++ /dev/null
@@ -1,285 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sqoop.manager.sqlserver;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.util.StringUtils;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
-import com.cloudera.sqoop.config.ConfigurationHelper;
-import com.cloudera.sqoop.orm.CompilationManager;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
-import com.cloudera.sqoop.testutil.ReparseMapper;
-import com.cloudera.sqoop.tool.ImportTool;
-import com.cloudera.sqoop.util.ClassLoaderStack;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.fail;
-
-/**
- * Test that the parse() methods generated in user SqoopRecord implementations
- * work in SQL Server.
- *
- * This uses JDBC to import data from an SQLServer database to HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerParseMethodsManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerParseMethodsManualTest extends ImportJobTestCase {
-
- @Before
- public void setUp() {
- super.setUp();
- Path p = new Path(getWarehouseDir());
- try {
- FileSystem fs = FileSystem.get(new Configuration());
- fs.delete(p);
- } catch (IOException e) {
- LOG.error("Setup fail with IOException: " + StringUtils.stringifyException(e));
- fail("Setup fail with IOException: " + StringUtils.stringifyException(e));
- }
- }
-
- @After
- public void tearDown() {
- try {
- dropTableIfExists(getTableName());
- } catch (SQLException sqle) {
- LOG.info("Table clean-up failed: " + sqle);
- } finally {
- super.tearDown();
- }
- }
-
- /**
- * Create the argv to pass to Sqoop.
- *
- * @return the argv as an array of strings.
- */
- private String[] getArgv(boolean includeHadoopFlags,
- String fieldTerminator, String lineTerminator, String encloser,
- String escape, boolean encloserRequired) {
-
- ArrayList<String> args = new ArrayList<String>();
-
- if (includeHadoopFlags) {
- CommonArgs.addHadoopFlags(args);
- }
-
- args.add("--table");
- args.add(getTableName());
- args.add("--warehouse-dir");
- args.add(getWarehouseDir());
- args.add("--connect");
- args.add(getConnectString());
- args.add("--as-textfile");
- args.add("--split-by");
- args.add("DATA_COL0"); // always split by first column.
- args.add("--fields-terminated-by");
- args.add(fieldTerminator);
- args.add("--lines-terminated-by");
- args.add(lineTerminator);
- args.add("--escaped-by");
- args.add(escape);
- if (encloserRequired) {
- args.add("--enclosed-by");
- } else {
- args.add("--optionally-enclosed-by");
- }
- args.add(encloser);
- args.add("--num-mappers");
- args.add("1");
-
- return args.toArray(new String[0]);
- }
-
- public void runParseTest(String fieldTerminator, String lineTerminator,
- String encloser, String escape, boolean encloseRequired)
- throws IOException {
-
- ClassLoader prevClassLoader = null;
-
- String[] argv = getArgv(true, fieldTerminator, lineTerminator,
- encloser, escape, encloseRequired);
- runImport(argv);
- try {
- String tableClassName = getTableName();
-
- argv = getArgv(false, fieldTerminator, lineTerminator, encloser,
- escape, encloseRequired);
- SqoopOptions opts = new ImportTool().parseArguments(argv, null,
- null, true);
-
- CompilationManager compileMgr = new CompilationManager(opts);
- String jarFileName = compileMgr.getJarFilename();
-
- // Make sure the user's class is loaded into our address space.
- prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
- tableClassName);
-
- JobConf job = new JobConf();
- job.setJar(jarFileName);
-
- // Tell the job what class we're testing.
- job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName);
-
- // use local mode in the same JVM.
- ConfigurationHelper.setJobtrackerAddr(job, "local");
- job.set("fs.default.name", "file:///");
-
- String warehouseDir = getWarehouseDir();
- Path warehousePath = new Path(warehouseDir);
- Path inputPath = new Path(warehousePath, getTableName());
- Path outputPath = new Path(warehousePath, getTableName() + "-out");
-
- job.setMapperClass(ReparseMapper.class);
- job.setNumReduceTasks(0);
- FileInputFormat.addInputPath(job, inputPath);
- FileOutputFormat.setOutputPath(job, outputPath);
-
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(NullWritable.class);
-
- JobClient.runJob(job);
- } catch (InvalidOptionsException ioe) {
- LOG.error(StringUtils.stringifyException(ioe));
- fail(ioe.toString());
- } catch (ParseException pe) {
- LOG.error(StringUtils.stringifyException(pe));
- fail(pe.toString());
- } finally {
- if (null != prevClassLoader) {
- ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
- }
- }
- }
-
- @Test
- public void testDefaults() throws IOException {
- String[] types = { "INTEGER", "VARCHAR(32)", "INTEGER" };
- String[] vals = { "64", "'foo'", "128" };
-
- createTableWithColTypes(types, vals);
- runParseTest(",", "\\n", "\\\"", "\\", false);
- }
-
- @Test
- public void testRequiredEnclose() throws IOException {
- String[] types = { "INTEGER", "VARCHAR(32)", "INTEGER" };
- String[] vals = { "64", "'foo'", "128" };
-
- createTableWithColTypes(types, vals);
- runParseTest(",", "\\n", "\\\"", "\\", true);
- }
-
- @Test
- public void testStringEscapes() throws IOException {
- String[] types = { "VARCHAR(32)", "VARCHAR(32)", "VARCHAR(32)",
- "VARCHAR(32)", "VARCHAR(32)", };
- String[] vals = { "'foo'", "'foo,bar'", "'foo''bar'", "'foo\\bar'",
- "'foo,bar''baz'", };
-
- createTableWithColTypes(types, vals);
- runParseTest(",", "\\n", "\\\'", "\\", false);
- }
-
- @Test
- public void testNumericTypes() throws IOException {
- String[] types = { "INTEGER", "REAL", "FLOAT", "DATE", "TIME", "BIT", };
- String[] vals = { "42", "36.0", "127.1", "'2009-07-02'", "'11:24:00'",
-
- "1", };
-
- createTableWithColTypes(types, vals);
- runParseTest(",", "\\n", "\\\'", "\\", false);
- }
-
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- protected String getConnectString() {
- return MSSQLTestUtils.getDBConnectString();
- }
-
- /**
- * Drop a table if it already exists in the database.
- *
- * @param table
- * the name of the table to drop.
- * @throws SQLException
- * if something goes wrong.
- */
- protected void dropTableIfExists(String table) throws SQLException {
- Connection conn = getManager().getConnection();
- String sqlStmt = "IF OBJECT_ID('" + table
- + "') IS NOT NULL DROP TABLE " + table;
-
- PreparedStatement statement = conn.prepareStatement(sqlStmt,
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- conn.commit();
- } finally {
- statement.close();
- }
- }
-
- protected SqoopOptions getSqoopOptions(Configuration conf) {
-
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- SqoopOptions opts = new SqoopOptions(conf);
- opts.setUsername(username);
- opts.setPassword(password);
- return opts;
-
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java
new file mode 100644
index 0000000..9547d80
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java
@@ -0,0 +1,287 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.sqlserver;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.util.StringUtils;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
+import com.cloudera.sqoop.config.ConfigurationHelper;
+import com.cloudera.sqoop.orm.CompilationManager;
+import com.cloudera.sqoop.testutil.CommonArgs;
+import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import com.cloudera.sqoop.testutil.ReparseMapper;
+import com.cloudera.sqoop.tool.ImportTool;
+import com.cloudera.sqoop.util.ClassLoaderStack;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
+
+/**
+ * Test that the parse() methods generated in user SqoopRecord implementations
+ * work in SQL Server.
+ *
+ * This uses JDBC to import data from an SQLServer database to HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerParseMethodsTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerParseMethodsTest extends ImportJobTestCase {
+
+ @Before
+ public void setUp() {
+ super.setUp();
+ Path p = new Path(getWarehouseDir());
+ try {
+ FileSystem fs = FileSystem.get(new Configuration());
+ fs.delete(p);
+ } catch (IOException e) {
+ LOG.error("Setup fail with IOException: " + StringUtils.stringifyException(e));
+ fail("Setup fail with IOException: " + StringUtils.stringifyException(e));
+ }
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ dropTableIfExists(getTableName());
+ } catch (SQLException sqle) {
+ LOG.info("Table clean-up failed: " + sqle);
+ } finally {
+ super.tearDown();
+ }
+ }
+
+ /**
+ * Create the argv to pass to Sqoop.
+ *
+ * @return the argv as an array of strings.
+ */
+ private String[] getArgv(boolean includeHadoopFlags,
+ String fieldTerminator, String lineTerminator, String encloser,
+ String escape, boolean encloserRequired) {
+
+ ArrayList<String> args = new ArrayList<String>();
+
+ if (includeHadoopFlags) {
+ CommonArgs.addHadoopFlags(args);
+ }
+
+ args.add("--table");
+ args.add(getTableName());
+ args.add("--warehouse-dir");
+ args.add(getWarehouseDir());
+ args.add("--connect");
+ args.add(getConnectString());
+ args.add("--as-textfile");
+ args.add("--split-by");
+ args.add("DATA_COL0"); // always split by first column.
+ args.add("--fields-terminated-by");
+ args.add(fieldTerminator);
+ args.add("--lines-terminated-by");
+ args.add(lineTerminator);
+ args.add("--escaped-by");
+ args.add(escape);
+ if (encloserRequired) {
+ args.add("--enclosed-by");
+ } else {
+ args.add("--optionally-enclosed-by");
+ }
+ args.add(encloser);
+ args.add("--num-mappers");
+ args.add("1");
+
+ return args.toArray(new String[0]);
+ }
+
+ public void runParseTest(String fieldTerminator, String lineTerminator,
+ String encloser, String escape, boolean encloseRequired)
+ throws IOException {
+
+ ClassLoader prevClassLoader = null;
+
+ String[] argv = getArgv(true, fieldTerminator, lineTerminator,
+ encloser, escape, encloseRequired);
+ runImport(argv);
+ try {
+ String tableClassName = getTableName();
+
+ argv = getArgv(false, fieldTerminator, lineTerminator, encloser,
+ escape, encloseRequired);
+ SqoopOptions opts = new ImportTool().parseArguments(argv, null,
+ null, true);
+
+ CompilationManager compileMgr = new CompilationManager(opts);
+ String jarFileName = compileMgr.getJarFilename();
+
+ // Make sure the user's class is loaded into our address space.
+ prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
+ tableClassName);
+
+ JobConf job = new JobConf();
+ job.setJar(jarFileName);
+
+ // Tell the job what class we're testing.
+ job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName);
+
+ // use local mode in the same JVM.
+ ConfigurationHelper.setJobtrackerAddr(job, "local");
+ job.set("fs.default.name", "file:///");
+
+ String warehouseDir = getWarehouseDir();
+ Path warehousePath = new Path(warehouseDir);
+ Path inputPath = new Path(warehousePath, getTableName());
+ Path outputPath = new Path(warehousePath, getTableName() + "-out");
+
+ job.setMapperClass(ReparseMapper.class);
+ job.setNumReduceTasks(0);
+ FileInputFormat.addInputPath(job, inputPath);
+ FileOutputFormat.setOutputPath(job, outputPath);
+
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(NullWritable.class);
+
+ JobClient.runJob(job);
+ } catch (InvalidOptionsException ioe) {
+ LOG.error(StringUtils.stringifyException(ioe));
+ fail(ioe.toString());
+ } catch (ParseException pe) {
+ LOG.error(StringUtils.stringifyException(pe));
+ fail(pe.toString());
+ } finally {
+ if (null != prevClassLoader) {
+ ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
+ }
+ }
+ }
+
+ @Test
+ public void testDefaults() throws IOException {
+ String[] types = { "INTEGER", "VARCHAR(32)", "INTEGER" };
+ String[] vals = { "64", "'foo'", "128" };
+
+ createTableWithColTypes(types, vals);
+ runParseTest(",", "\\n", "\\\"", "\\", false);
+ }
+
+ @Test
+ public void testRequiredEnclose() throws IOException {
+ String[] types = { "INTEGER", "VARCHAR(32)", "INTEGER" };
+ String[] vals = { "64", "'foo'", "128" };
+
+ createTableWithColTypes(types, vals);
+ runParseTest(",", "\\n", "\\\"", "\\", true);
+ }
+
+ @Test
+ public void testStringEscapes() throws IOException {
+ String[] types = { "VARCHAR(32)", "VARCHAR(32)", "VARCHAR(32)",
+ "VARCHAR(32)", "VARCHAR(32)", };
+ String[] vals = { "'foo'", "'foo,bar'", "'foo''bar'", "'foo\\bar'",
+ "'foo,bar''baz'", };
+
+ createTableWithColTypes(types, vals);
+ runParseTest(",", "\\n", "\\\'", "\\", false);
+ }
+
+ @Test
+ public void testNumericTypes() throws IOException {
+ String[] types = { "INTEGER", "REAL", "FLOAT", "DATE", "TIME", "BIT", };
+ String[] vals = { "42", "36.0", "127.1", "'2009-07-02'", "'11:24:00'",
+
+ "1", };
+
+ createTableWithColTypes(types, vals);
+ runParseTest(",", "\\n", "\\\'", "\\", false);
+ }
+
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ protected String getConnectString() {
+ return MSSQLTestUtils.getDBConnectString();
+ }
+
+ /**
+ * Drop a table if it already exists in the database.
+ *
+ * @param table
+ * the name of the table to drop.
+ * @throws SQLException
+ * if something goes wrong.
+ */
+ protected void dropTableIfExists(String table) throws SQLException {
+ Connection conn = getManager().getConnection();
+ String sqlStmt = "IF OBJECT_ID('" + table
+ + "') IS NOT NULL DROP TABLE " + table;
+
+ PreparedStatement statement = conn.prepareStatement(sqlStmt,
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ statement.executeUpdate();
+ conn.commit();
+ } finally {
+ statement.close();
+ }
+ }
+
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ SqoopOptions opts = new SqoopOptions(conf);
+ opts.setUsername(username);
+ opts.setPassword(password);
+ return opts;
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerQueryManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerQueryManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerQueryManualTest.java
deleted file mode 100644
index d891c2b..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerQueryManualTest.java
+++ /dev/null
@@ -1,301 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sqoop.manager.sqlserver;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
-import com.cloudera.sqoop.orm.CompilationManager;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
-import com.cloudera.sqoop.testutil.SeqFileReader;
-import com.cloudera.sqoop.tool.ImportTool;
-import com.cloudera.sqoop.util.ClassLoaderStack;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * Test that --query works in SQL Server.
- *
- * This uses JDBC to import data from an SQLServer database to HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerQueryManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerQueryManualTest extends ImportJobTestCase {
-
- @Before
- public void setUp() {
- super.setUp();
- MSSQLTestUtils utils = new MSSQLTestUtils();
- try {
- utils.createTableFromSQL(MSSQLTestUtils.CREATE_TALBE_LINEITEM);
- utils.populateLineItem();
- } catch (SQLException e) {
- LOG.error("Setup fail with SQLException: " + StringUtils.stringifyException(e));
- fail("Setup fail with SQLException: " + e.toString());
- }
-
- }
-
- @After
- public void tearDown() {
- super.tearDown();
- MSSQLTestUtils utils = new MSSQLTestUtils();
- try {
- utils.dropTableIfExists("TPCH1M_LINEITEM");
- } catch (SQLException e) {
- LOG.error("TearDown fail with SQLException: " + StringUtils.stringifyException(e));
- fail("TearDown fail with SQLException: " + e.toString());
- }
- }
-
- /**
- * Create the argv to pass to Sqoop.
- *
- * @return the argv as an array of strings.
- */
- protected String[] getArgv(boolean includeHadoopFlags, String query,
- String targetDir, boolean allowParallel) {
-
- ArrayList<String> args = new ArrayList<String>();
-
- if (includeHadoopFlags) {
- CommonArgs.addHadoopFlags(args);
- }
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
-
- args.add("--query");
- args.add(query);
- args.add("--split-by");
- args.add("L_ORDERKEY");
- args.add("--connect");
- args.add(getConnectString());
- args.add("--username");
- args.add(username);
- args.add("--password");
- args.add(password);
- args.add("--as-sequencefile");
- args.add("--target-dir");
- args.add(targetDir);
- args.add("--class-name");
- args.add(getTableName());
- if (allowParallel) {
- args.add("--num-mappers");
- args.add("2");
- } else {
- args.add("--num-mappers");
- args.add("1");
- }
-
- return args.toArray(new String[0]);
- }
-
- /**
- * Given a comma-delimited list of integers, grab and parse the first int.
- *
- * @param str
- * a comma-delimited list of values, the first of which is an
- * int.
- * @return the first field in the string, cast to int
- */
- private int getFirstInt(String str) {
- String[] parts = str.split(",");
- return Integer.parseInt(parts[0]);
- }
-
- public void runQueryTest(String query, String firstValStr,
- int numExpectedResults, int expectedSum, String targetDir)
- throws IOException {
-
- ClassLoader prevClassLoader = null;
- SequenceFile.Reader reader = null;
-
- String[] argv = getArgv(true, query, targetDir, false);
- runImport(argv);
- try {
- SqoopOptions opts = new ImportTool().parseArguments(getArgv(false,
- query, targetDir, false), null, null, true);
-
- CompilationManager compileMgr = new CompilationManager(opts);
- String jarFileName = compileMgr.getJarFilename();
-
- prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
- getTableName());
-
- reader = SeqFileReader.getSeqFileReader(getDataFilePath()
- .toString());
-
- // here we can actually instantiate (k, v) pairs.
- Configuration conf = new Configuration();
- Object key = ReflectionUtils
- .newInstance(reader.getKeyClass(), conf);
- Object val = ReflectionUtils.newInstance(reader.getValueClass(),
- conf);
-
- if (reader.next(key) == null) {
- fail("Empty SequenceFile during import");
- }
-
- // make sure that the value we think should be at the top, is.
- reader.getCurrentValue(val);
- assertEquals("Invalid ordering within sorted SeqFile", firstValStr,
- val.toString());
-
- // We know that these values are two ints separated by a ','
- // character.
- // Since this is all dynamic, though, we don't want to actually link
- // against the class and use its methods. So we just parse this back
- // into int fields manually. Sum them up and ensure that we get the
- // expected total for the first column, to verify that we got all
- // the
- // results from the db into the file.
- int curSum = getFirstInt(val.toString());
- int totalResults = 1;
-
- // now sum up everything else in the file.
- while (reader.next(key) != null) {
- reader.getCurrentValue(val);
- curSum += getFirstInt(val.toString());
- totalResults++;
- }
-
- assertEquals("Total sum of first db column mismatch", expectedSum,
- curSum);
- assertEquals("Incorrect number of results for query",
- numExpectedResults, totalResults);
- } catch (InvalidOptionsException ioe) {
- LOG.error(StringUtils.stringifyException(ioe));
- fail(ioe.toString());
- } catch (ParseException pe) {
- LOG.error(StringUtils.stringifyException(pe));
- fail(pe.toString());
- } finally {
- IOUtils.closeStream(reader);
-
- if (null != prevClassLoader) {
- ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
- }
- }
- }
-
- @Test
- public void testSelectStar() throws IOException {
- runQueryTest("SELECT * FROM " + getTableName()
- + " WHERE L_ORDERKEY > 0 AND $CONDITIONS",
- "1,2,3,4,5,6.00,7.00,8.00,AB,CD,abcd,efgh,hijk,dothis,likethis,"
- + "nocomments\n", 4, 10, getTablePath().toString());
- }
-
- @Test
- public void testCompoundWhere() throws IOException {
- runQueryTest("SELECT * FROM " + getTableName()
- + " WHERE L_ORDERKEY > 1 AND L_PARTKEY < 4 AND $CONDITIONS",
- "2,3,4,5,6,7.00,8.00,9.00,AB,CD,abcd,efgh,hijk,dothis,likethis,"
- + "nocomments\n", 1, 2, getTablePath().toString());
- }
-
- @Test
- public void testFailNoConditions() throws IOException {
- String[] argv = getArgv(true, "SELECT * FROM " + getTableName(),
- getTablePath().toString() + "where $CONDITIONS", true);
- try {
- runImport(argv);
- fail("Expected exception running import without $CONDITIONS");
- } catch (Exception e) {
- LOG.info("Got exception " + e + " running job (expected; ok)");
- }
- }
-
- protected boolean useHsqldbTestServer() {
-
- return false;
- }
-
- protected String getConnectString() {
- return MSSQLTestUtils.getDBConnectString();
- }
-
- /**
- * Drop a table if it already exists in the database.
- *
- * @param table
- * the name of the table to drop.
- * @throws SQLException
- * if something goes wrong.
- */
- protected void dropTableIfExists(String table) throws SQLException {
- Connection conn = getManager().getConnection();
- String sqlStmt = "IF OBJECT_ID('" + table
- + "') IS NOT NULL DROP TABLE " + table;
-
- PreparedStatement statement = conn.prepareStatement(sqlStmt,
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- conn.commit();
- } finally {
- statement.close();
- }
- }
-
- protected SqoopOptions getSqoopOptions(Configuration conf) {
- SqoopOptions opt = new SqoopOptions(conf);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- SqoopOptions opts = new SqoopOptions(conf);
- opts.setUsername(username);
- opts.setPassword(password);
-
- return opt;
- }
-
- protected String getTableName() {
- return "tpch1m_lineitem";
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerQueryTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerQueryTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerQueryTest.java
new file mode 100644
index 0000000..1d570fe
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerQueryTest.java
@@ -0,0 +1,303 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.sqlserver;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
+import com.cloudera.sqoop.orm.CompilationManager;
+import com.cloudera.sqoop.testutil.CommonArgs;
+import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import com.cloudera.sqoop.testutil.SeqFileReader;
+import com.cloudera.sqoop.tool.ImportTool;
+import com.cloudera.sqoop.util.ClassLoaderStack;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Test that --query works in SQL Server.
+ *
+ * This uses JDBC to import data from an SQLServer database to HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerQueryTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerQueryTest extends ImportJobTestCase {
+
+ @Before
+ public void setUp() {
+ super.setUp();
+ MSSQLTestUtils utils = new MSSQLTestUtils();
+ try {
+ utils.createTableFromSQL(MSSQLTestUtils.CREATE_TALBE_LINEITEM);
+ utils.populateLineItem();
+ } catch (SQLException e) {
+ LOG.error("Setup fail with SQLException: " + StringUtils.stringifyException(e));
+ fail("Setup fail with SQLException: " + e.toString());
+ }
+
+ }
+
+ @After
+ public void tearDown() {
+ super.tearDown();
+ MSSQLTestUtils utils = new MSSQLTestUtils();
+ try {
+ utils.dropTableIfExists("TPCH1M_LINEITEM");
+ } catch (SQLException e) {
+ LOG.error("TearDown fail with SQLException: " + StringUtils.stringifyException(e));
+ fail("TearDown fail with SQLException: " + e.toString());
+ }
+ }
+
+ /**
+ * Create the argv to pass to Sqoop.
+ *
+ * @return the argv as an array of strings.
+ */
+ protected String[] getArgv(boolean includeHadoopFlags, String query,
+ String targetDir, boolean allowParallel) {
+
+ ArrayList<String> args = new ArrayList<String>();
+
+ if (includeHadoopFlags) {
+ CommonArgs.addHadoopFlags(args);
+ }
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+
+ args.add("--query");
+ args.add(query);
+ args.add("--split-by");
+ args.add("L_ORDERKEY");
+ args.add("--connect");
+ args.add(getConnectString());
+ args.add("--username");
+ args.add(username);
+ args.add("--password");
+ args.add(password);
+ args.add("--as-sequencefile");
+ args.add("--target-dir");
+ args.add(targetDir);
+ args.add("--class-name");
+ args.add(getTableName());
+ if (allowParallel) {
+ args.add("--num-mappers");
+ args.add("2");
+ } else {
+ args.add("--num-mappers");
+ args.add("1");
+ }
+
+ return args.toArray(new String[0]);
+ }
+
+ /**
+ * Given a comma-delimited list of integers, grab and parse the first int.
+ *
+ * @param str
+ * a comma-delimited list of values, the first of which is an
+ * int.
+ * @return the first field in the string, cast to int
+ */
+ private int getFirstInt(String str) {
+ String[] parts = str.split(",");
+ return Integer.parseInt(parts[0]);
+ }
+
+ public void runQueryTest(String query, String firstValStr,
+ int numExpectedResults, int expectedSum, String targetDir)
+ throws IOException {
+
+ ClassLoader prevClassLoader = null;
+ SequenceFile.Reader reader = null;
+
+ String[] argv = getArgv(true, query, targetDir, false);
+ runImport(argv);
+ try {
+ SqoopOptions opts = new ImportTool().parseArguments(getArgv(false,
+ query, targetDir, false), null, null, true);
+
+ CompilationManager compileMgr = new CompilationManager(opts);
+ String jarFileName = compileMgr.getJarFilename();
+
+ prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
+ getTableName());
+
+ reader = SeqFileReader.getSeqFileReader(getDataFilePath()
+ .toString());
+
+ // here we can actually instantiate (k, v) pairs.
+ Configuration conf = new Configuration();
+ Object key = ReflectionUtils
+ .newInstance(reader.getKeyClass(), conf);
+ Object val = ReflectionUtils.newInstance(reader.getValueClass(),
+ conf);
+
+ if (reader.next(key) == null) {
+ fail("Empty SequenceFile during import");
+ }
+
+ // make sure that the value we think should be at the top, is.
+ reader.getCurrentValue(val);
+ assertEquals("Invalid ordering within sorted SeqFile", firstValStr,
+ val.toString());
+
+ // We know that these values are two ints separated by a ','
+ // character.
+ // Since this is all dynamic, though, we don't want to actually link
+ // against the class and use its methods. So we just parse this back
+ // into int fields manually. Sum them up and ensure that we get the
+ // expected total for the first column, to verify that we got all
+ // the
+ // results from the db into the file.
+ int curSum = getFirstInt(val.toString());
+ int totalResults = 1;
+
+ // now sum up everything else in the file.
+ while (reader.next(key) != null) {
+ reader.getCurrentValue(val);
+ curSum += getFirstInt(val.toString());
+ totalResults++;
+ }
+
+ assertEquals("Total sum of first db column mismatch", expectedSum,
+ curSum);
+ assertEquals("Incorrect number of results for query",
+ numExpectedResults, totalResults);
+ } catch (InvalidOptionsException ioe) {
+ LOG.error(StringUtils.stringifyException(ioe));
+ fail(ioe.toString());
+ } catch (ParseException pe) {
+ LOG.error(StringUtils.stringifyException(pe));
+ fail(pe.toString());
+ } finally {
+ IOUtils.closeStream(reader);
+
+ if (null != prevClassLoader) {
+ ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
+ }
+ }
+ }
+
+ @Test
+ public void testSelectStar() throws IOException {
+ runQueryTest("SELECT * FROM " + getTableName()
+ + " WHERE L_ORDERKEY > 0 AND $CONDITIONS",
+ "1,2,3,4,5,6.00,7.00,8.00,AB,CD,abcd,efgh,hijk,dothis,likethis,"
+ + "nocomments\n", 4, 10, getTablePath().toString());
+ }
+
+ @Test
+ public void testCompoundWhere() throws IOException {
+ runQueryTest("SELECT * FROM " + getTableName()
+ + " WHERE L_ORDERKEY > 1 AND L_PARTKEY < 4 AND $CONDITIONS",
+ "2,3,4,5,6,7.00,8.00,9.00,AB,CD,abcd,efgh,hijk,dothis,likethis,"
+ + "nocomments\n", 1, 2, getTablePath().toString());
+ }
+
+ @Test
+ public void testFailNoConditions() throws IOException {
+ String[] argv = getArgv(true, "SELECT * FROM " + getTableName(),
+ getTablePath().toString() + "where $CONDITIONS", true);
+ try {
+ runImport(argv);
+ fail("Expected exception running import without $CONDITIONS");
+ } catch (Exception e) {
+ LOG.info("Got exception " + e + " running job (expected; ok)");
+ }
+ }
+
+ protected boolean useHsqldbTestServer() {
+
+ return false;
+ }
+
+ protected String getConnectString() {
+ return MSSQLTestUtils.getDBConnectString();
+ }
+
+ /**
+ * Drop a table if it already exists in the database.
+ *
+ * @param table
+ * the name of the table to drop.
+ * @throws SQLException
+ * if something goes wrong.
+ */
+ protected void dropTableIfExists(String table) throws SQLException {
+ Connection conn = getManager().getConnection();
+ String sqlStmt = "IF OBJECT_ID('" + table
+ + "') IS NOT NULL DROP TABLE " + table;
+
+ PreparedStatement statement = conn.prepareStatement(sqlStmt,
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ statement.executeUpdate();
+ conn.commit();
+ } finally {
+ statement.close();
+ }
+ }
+
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ SqoopOptions opt = new SqoopOptions(conf);
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ SqoopOptions opts = new SqoopOptions(conf);
+ opts.setUsername(username);
+ opts.setPassword(password);
+
+ return opt;
+ }
+
+ protected String getTableName() {
+ return "tpch1m_lineitem";
+ }
+}