You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by an...@apache.org on 2017/05/15 08:10:11 UTC
[3/4] sqoop git commit: SQOOP-3174: Add SQLServer manual tests to 3rd
party test suite
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java
deleted file mode 100644
index 9c20bca..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileManualTest.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.manager.sqlserver;
-
-import java.io.BufferedReader;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES;
-import com.cloudera.sqoop.Sqoop;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.orm.CompilationManager;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.tool.ImportTool;
-import com.cloudera.sqoop.util.ClassLoaderStack;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * Test to import delimited file from SQL Server.
- *
- * This uses JDBC to import data from an SQLServer database to HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerDatatypeImportDelimitedFileManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerDatatypeImportDelimitedFileManualTest
- extends SQLServerDatatypeImportSequenceFileManualTest {
-
-/**
- * Create the argv to pass to Sqoop.
- *
- * @param includeHadoopFlags
- * if true, then include -D various.settings=values
- * @param colNames
- * the columns to import. If null, all columns are used.
- * @param conf
- * a Configuration specifying additional properties to use when
- * determining the arguments.
- * @return the argv as an array of strings.
-*/
- protected String[] getArgv(boolean includeHadoopFlags, String[] colNames,
- Configuration conf) {
- if (null == colNames) {
- colNames = getColNames();
- }
-
- String splitByCol = colNames[0];
- String columnsString = "";
- for (String col : colNames) {
- columnsString += col + ",";
- }
-
- ArrayList<String> args = new ArrayList<String>();
-
- if (includeHadoopFlags) {
- CommonArgs.addHadoopFlags(args);
- }
-
- args.add("--table");
- args.add(getTableName());
- args.add("--columns");
- args.add(columnsString);
- args.add("--split-by");
- args.add(splitByCol);
- args.add("--warehouse-dir");
- args.add(getWarehouseDir());
- args.add("--connect");
- args.add(MSSQLTestUtils.getDBConnectString());
-
- args.add("--num-mappers");
- args.add("2");
-
- args.addAll(getExtraArgs(conf));
-
- return args.toArray(new String[0]);
- }
-
-
- private void runSqoopImport(String[] importCols) {
- Configuration conf = getConf();
- SqoopOptions opts = getSqoopOptions(conf);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- opts.setUsername(username);
- opts.setPassword(password);
-
- // run the tool through the normal entry-point.
- int ret;
- try {
- Sqoop importer = new Sqoop(new ImportTool(), conf, opts);
- ret = Sqoop.runSqoop(importer, getArgv(true, importCols, conf));
- } catch (Exception e) {
- LOG.error("Got exception running Sqoop: " + e.toString());
- throw new RuntimeException(e);
- }
-
- // expect a successful return.
- assertEquals("Failure during job", 0, ret);
- }
-
- /**
- * Do a MapReduce-based import of the table and verify that the results were
- * imported as expected. (tests readFields(ResultSet) and toString())
- *
- * @param expectedVal
- * the value we injected into the table.
- * @param importCols
- * the columns to import. If null, all columns are used.
- */
- protected void verifyImport(String expectedVal, String[] importCols) {
-
- // paths to where our output file will wind up.
- Path tableDirPath = getTablePath();
-
- removeTableDir();
-
- runSqoopImport(importCols);
- Configuration conf = getConf();
-
- SqoopOptions opts = getSqoopOptions(conf);
- try {
- ImportTool importTool = new ImportTool();
- opts = importTool.parseArguments(getArgv(false, importCols, conf),
- conf, opts, true);
- } catch (Exception e) {
- LOG.error(StringUtils.stringifyException(e));
- fail(e.toString());
- }
-
- CompilationManager compileMgr = new CompilationManager(opts);
- String jarFileName = compileMgr.getJarFilename();
- ClassLoader prevClassLoader = null;
- try {
- prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
- getTableName());
-
- // Now open and check all part-files in the table path until we find
- // a non-empty one that we can verify contains the value.
-
- FileSystem fs = FileSystem.getLocal(conf);
- FileStatus[] stats = fs.listStatus(tableDirPath);
-
- if (stats == null || stats.length == 0) {
- fail("Error: no files in " + tableDirPath);
- }
-
- boolean foundRecord = false;
- for (FileStatus stat : stats) {
- if (!stat.getPath().getName().startsWith("part-")
- && !stat.getPath().getName().startsWith("data-")) {
- // This isn't a data file. Ignore it.
- continue;
- }
-
- try {
- String line;
- String fname = stat.getPath().toString();
- fname = fname.substring(5, fname.length());
-
- BufferedReader reader = new BufferedReader(
- new InputStreamReader(new FileInputStream(new File(
- fname))));
- try {
- line = reader.readLine();
- assertEquals(" expected a different string",
- expectedVal, line);
- } finally {
- IOUtils.closeStream(reader);
- }
- LOG.info("Read back from delimited file: " + line);
- foundRecord = true;
- // Add trailing '\n' to expected value since
- // SqoopRecord.toString()
- // encodes the record delim.
- if (null == expectedVal) {
- assertEquals("Error validating result from delimited file",
- "null\n", line);
- }
- } catch (EOFException eoe) {
- // EOF in a file isn't necessarily a problem. We may have
- // some
- // empty sequence files, which will throw this. Just
- // continue
- // in the loop.
- }
- }
-
- if (!foundRecord) {
- fail("Couldn't read any records from delimited file");
- }
- } catch (IOException ioe) {
- LOG.error(StringUtils.stringifyException(ioe));
- fail("IOException: " + ioe.toString());
- } finally {
- if (null != prevClassLoader) {
- ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
- }
- }
- }
-
- @Test
- public void testTime() {
- if (!supportsTime()) {
- skipped = true;
- return;
- }
-
- dataTypeTest(DATATYPES.TIME);
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testVarBinary() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testBit() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testBit2() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testBit3() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testNChar() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testChar() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testVarchar() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testNVarchar() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testBinary() {
- }
-
- @Ignore("Ignored as used type is not supported for table splitting.")
- @Test
- public void testTimestamp3() {
- }
-
- public String getResportFileName(){
- return this.getClass().toString()+".txt";
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileTest.java
new file mode 100644
index 0000000..a4d1822
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportDelimitedFileTest.java
@@ -0,0 +1,306 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.manager.sqlserver;
+
+import java.io.BufferedReader;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES;
+import com.cloudera.sqoop.Sqoop;
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.orm.CompilationManager;
+import com.cloudera.sqoop.testutil.CommonArgs;
+import com.cloudera.sqoop.tool.ImportTool;
+import com.cloudera.sqoop.util.ClassLoaderStack;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Test to import delimited file from SQL Server.
+ *
+ * This uses JDBC to import data from an SQLServer database to HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerDatatypeImportDelimitedFileTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerDatatypeImportDelimitedFileTest
+ extends SQLServerDatatypeImportSequenceFileTest {
+
+/**
+ * Create the argv to pass to Sqoop.
+ *
+ * @param includeHadoopFlags
+ * if true, then include -D various.settings=values
+ * @param colNames
+ * the columns to import. If null, all columns are used.
+ * @param conf
+ * a Configuration specifying additional properties to use when
+ * determining the arguments.
+ * @return the argv as an array of strings.
+*/
+ protected String[] getArgv(boolean includeHadoopFlags, String[] colNames,
+ Configuration conf) {
+ if (null == colNames) {
+ colNames = getColNames();
+ }
+
+ String splitByCol = colNames[0];
+ String columnsString = "";
+ for (String col : colNames) {
+ columnsString += col + ",";
+ }
+
+ ArrayList<String> args = new ArrayList<String>();
+
+ if (includeHadoopFlags) {
+ CommonArgs.addHadoopFlags(args);
+ }
+
+ args.add("--table");
+ args.add(getTableName());
+ args.add("--columns");
+ args.add(columnsString);
+ args.add("--split-by");
+ args.add(splitByCol);
+ args.add("--warehouse-dir");
+ args.add(getWarehouseDir());
+ args.add("--connect");
+ args.add(MSSQLTestUtils.getDBConnectString());
+
+ args.add("--num-mappers");
+ args.add("2");
+
+ args.addAll(getExtraArgs(conf));
+
+ return args.toArray(new String[0]);
+ }
+
+
+ private void runSqoopImport(String[] importCols) {
+ Configuration conf = getConf();
+ SqoopOptions opts = getSqoopOptions(conf);
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ opts.setUsername(username);
+ opts.setPassword(password);
+
+ // run the tool through the normal entry-point.
+ int ret;
+ try {
+ Sqoop importer = new Sqoop(new ImportTool(), conf, opts);
+ ret = Sqoop.runSqoop(importer, getArgv(true, importCols, conf));
+ } catch (Exception e) {
+ LOG.error("Got exception running Sqoop: " + e.toString());
+ throw new RuntimeException(e);
+ }
+
+ // expect a successful return.
+ assertEquals("Failure during job", 0, ret);
+ }
+
+ /**
+ * Do a MapReduce-based import of the table and verify that the results were
+ * imported as expected. (tests readFields(ResultSet) and toString())
+ *
+ * @param expectedVal
+ * the value we injected into the table.
+ * @param importCols
+ * the columns to import. If null, all columns are used.
+ */
+ protected void verifyImport(String expectedVal, String[] importCols) {
+
+ // paths to where our output file will wind up.
+ Path tableDirPath = getTablePath();
+
+ removeTableDir();
+
+ runSqoopImport(importCols);
+ Configuration conf = getConf();
+
+ SqoopOptions opts = getSqoopOptions(conf);
+ try {
+ ImportTool importTool = new ImportTool();
+ opts = importTool.parseArguments(getArgv(false, importCols, conf),
+ conf, opts, true);
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ fail(e.toString());
+ }
+
+ CompilationManager compileMgr = new CompilationManager(opts);
+ String jarFileName = compileMgr.getJarFilename();
+ ClassLoader prevClassLoader = null;
+ try {
+ prevClassLoader = ClassLoaderStack.addJarFile(jarFileName,
+ getTableName());
+
+ // Now open and check all part-files in the table path until we find
+ // a non-empty one that we can verify contains the value.
+
+ FileSystem fs = FileSystem.getLocal(conf);
+ FileStatus[] stats = fs.listStatus(tableDirPath);
+
+ if (stats == null || stats.length == 0) {
+ fail("Error: no files in " + tableDirPath);
+ }
+
+ boolean foundRecord = false;
+ for (FileStatus stat : stats) {
+ if (!stat.getPath().getName().startsWith("part-")
+ && !stat.getPath().getName().startsWith("data-")) {
+ // This isn't a data file. Ignore it.
+ continue;
+ }
+
+ try {
+ String line;
+ String fname = stat.getPath().toString();
+ fname = fname.substring(5, fname.length());
+
+ BufferedReader reader = new BufferedReader(
+ new InputStreamReader(new FileInputStream(new File(
+ fname))));
+ try {
+ line = reader.readLine();
+ assertEquals(" expected a different string",
+ expectedVal, line);
+ } finally {
+ IOUtils.closeStream(reader);
+ }
+ LOG.info("Read back from delimited file: " + line);
+ foundRecord = true;
+ // Add trailing '\n' to expected value since
+ // SqoopRecord.toString()
+ // encodes the record delim.
+ if (null == expectedVal) {
+ assertEquals("Error validating result from delimited file",
+ "null\n", line);
+ }
+ } catch (EOFException eoe) {
+ // EOF in a file isn't necessarily a problem. We may have
+ // some
+ // empty sequence files, which will throw this. Just
+ // continue
+ // in the loop.
+ }
+ }
+
+ if (!foundRecord) {
+ fail("Couldn't read any records from delimited file");
+ }
+ } catch (IOException ioe) {
+ LOG.error(StringUtils.stringifyException(ioe));
+ fail("IOException: " + ioe.toString());
+ } finally {
+ if (null != prevClassLoader) {
+ ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
+ }
+ }
+ }
+
+ @Test
+ public void testTime() {
+ if (!supportsTime()) {
+ skipped = true;
+ return;
+ }
+
+ dataTypeTest(DATATYPES.TIME);
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testVarBinary() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testBit() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testBit2() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testBit3() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testNChar() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testChar() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testVarchar() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testNVarchar() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testBinary() {
+ }
+
+ @Ignore("Ignored as used type is not supported for table splitting.")
+ @Test
+ public void testTimestamp3() {
+ }
+
+ public String getResportFileName(){
+ return this.getClass().toString()+".txt";
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java
deleted file mode 100644
index 9cd3176..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileManualTest.java
+++ /dev/null
@@ -1,845 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.manager.sqlserver;
-
-import java.io.FileWriter;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.StringUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.testutil.ManagerCompatTestCase;
-import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES;
-import org.apache.sqoop.manager.sqlserver.MSSQLTestData.KEY_STRINGS;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Test importing sequence file from SQL Server.
- *
- * This uses JDBC to import data from an SQLServer database to HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerDatatypeImportSequenceFileManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerDatatypeImportSequenceFileManualTest extends
- ManagerCompatTestCase {
-
- public static final Log LOG = LogFactory.getLog(
- SQLServerDatatypeImportSequenceFileManualTest.class.getName());
- private static MSSQLTestDataFileParser tdfs;
- private static Map report;
-
- static {
- try {
-
- String testfile = null;
- testfile = System.getProperty("test.data.dir")
- + "/" + System.getProperty("ms.datatype.test.data.file.import");
- String delim = System.getProperty("ms.datatype.test.data.file.delim", ",");
- System.out.println("Using data file : " + testfile);
- LOG.info("Using data file : " + testfile);
- tdfs = new MSSQLTestDataFileParser(testfile);
- tdfs.setDelim(delim);
- tdfs.parse();
- report = new HashMap();
- } catch (Exception e) {
- LOG.error(StringUtils.stringifyException(e));
- System.out
- .println("Error with test data file, check stack trace for cause"
- + ".\nTests cannont continue.");
- System.exit(0);
- }
- }
-
- @Override
- protected String getDbFriendlyName() {
- return "MSSQL";
- }
-
- @Override
- protected Log getLogger() {
- return LOG;
- }
-
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- protected String getConnectString() {
- return MSSQLTestUtils.getDBConnectString();
- }
-
- /**
- * Drop a table if it already exists in the database.
- *
- * @param table
- * the name of the table to drop.
- * @throws SQLException
- * if something goes wrong.
- */
- protected void dropTableIfExists(String table) throws SQLException {
- Connection conn = getManager().getConnection();
- String sqlStmt = "IF OBJECT_ID('" + table
- + "') IS NOT NULL DROP TABLE " + table;
-
- PreparedStatement statement = conn.prepareStatement(sqlStmt,
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- conn.commit();
- } finally {
- statement.close();
- }
- }
-
- protected SqoopOptions getSqoopOptions(Configuration conf) {
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- SqoopOptions opts = new SqoopOptions(conf);
- opts.setUsername(username);
- opts.setPassword(password);
- return opts;
- }
-
- @Before
- public void setUp() {
- try {
- super.setUp();
- } catch (Exception e) {
- try {
- FileWriter fr = new FileWriter(getResportFileName(), true);
- String res = removeNewLines(e.getMessage());
- fr.append("Error\t" + res + "\n");
- fr.close();
- } catch (Exception e2) {
- LOG.error(StringUtils.stringifyException(e2));
- fail(e2.toString());
- }
- } catch (Error e) {
- try {
- FileWriter fr = new FileWriter(getResportFileName(), true);
-
- String res = removeNewLines(e.getMessage());
-
- fr.append("Error\t" + res + "\n");
- fr.close();
- fail(res);
- } catch (Exception e2) {
- LOG.error(StringUtils.stringifyException(e2));
- fail(e2.toString());
- }
- }
- }
-
- @After
- public void tearDown() {
- try {
- super.tearDown();
- } catch (Exception e) {
- try {
- FileWriter fr = new FileWriter(getResportFileName(), true);
- String res = removeNewLines(e.getMessage());
- fr.append("Error\t" + res + "\n");
- fr.close();
- } catch (Exception e2) {
- LOG.error(StringUtils.stringifyException(e2));
- fail(e2.toString());
- }
- } catch (Error e) {
- try {
- FileWriter fr = new FileWriter(getResportFileName(), true);
- String res = removeNewLines(e.getMessage());
- fr.append("Error\t" + res + "\n");
- fr.close();
- fail(res);
- } catch (Exception e2) {
- LOG.error(StringUtils.stringifyException(e2));
- fail(e2.toString());
- }
- }
- }
-
- protected boolean supportsBoolean() {
- return true;
- }
-
- @Test
- public void testBit() {
- if (!supportsBoolean()) {
- skipped = true;
- return;
- }
- verifyType("BIT", getTrueBoolNumericSqlInput(), getTrueBoolSeqOutput());
- }
-
- @Test
- public void testBit2() {
- if (!supportsBoolean()) {
- skipped = true;
- return;
- }
- verifyType("BIT", getFalseBoolNumericSqlInput(), getFalseBoolSeqOutput());
- }
-
- @Test
- public void testBit3() {
- if (!supportsBoolean()) {
- skipped = true;
- return;
- }
- verifyType("BIT", getFalseBoolLiteralSqlInput(), getFalseBoolSeqOutput());
- }
-
- @Test
- public void testBoolean() {
- try {
- super.testBoolean();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testBoolean2() {
- try {
- super.testBoolean2();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testBoolean3() {
- try {
- super.testBoolean3();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testDouble1() {
- try {
- super.testDouble1();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testDouble2() {
- try {
- super.testDouble2();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testClob1() {
- try {
- super.testClob1();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testBlob1() {
- try {
- super.testBlob1();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testLongVarChar() {
- try {
- super.testLongVarChar();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testTimestamp1() {
- try {
- super.testTimestamp1();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testTimestamp2() {
- try {
- super.testTimestamp2();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testTimestamp3() {
- try {
- super.testTimestamp3();
- assertTrue("This test should not pass on sql server", false);
- } catch (AssertionError a) {
- System.out.println("Test failed, this was expected");
- }
- }
-
- @Test
- public void testVarBinary() {
- if (!supportsVarBinary()) {
- return;
- }
- dataTypeTest(DATATYPES.VARBINARY);
- }
-
- @Test
- public void testTime() {
- if (!supportsTime()) {
- skipped = true;
- return;
- }
- dataTypeTest(DATATYPES.TIME);
- }
-
- @Test
- public void testSmalldatetime() {
- if (!supportsTime()) {
- skipped = true;
- return;
- }
- dataTypeTest(DATATYPES.SMALLDATETIME);
- }
-
- @Test
- public void testdatetime2() {
- if (!supportsTime()) {
- skipped = true;
- return;
- }
- dataTypeTest(DATATYPES.DATETIME2);
- }
-
- @Test
- public void testdatetime() {
- if (!supportsTime()) {
- skipped = true;
- return;
- }
- dataTypeTest(DATATYPES.DATETIME);
- }
-
- @Test
- public void testdatetimeoffset() {
- if (!supportsTime()) {
- skipped = true;
- return;
- }
- dataTypeTest(DATATYPES.DATETIMEOFFSET);
- }
-
- @Test
- public void testDecimal() {
- dataTypeTest(DATATYPES.DECIMAL);
- }
-
- @Test
- public void testNumeric() {
- dataTypeTest(DATATYPES.NUMERIC);
- }
-
- @Test
- public void testNumeric1() {
- }
-
- @Test
- public void testNumeric2() {
- }
-
- @Test
- public void testDecimal1() {
- }
-
- @Test
- public void testDecimal2() {
- }
-
- @Test
- public void testBigInt() {
- dataTypeTest(DATATYPES.BIGINT);
- }
-
- @Test
- public void testBigInt1() {
- }
-
- @Test
- public void testInt() {
- dataTypeTest(DATATYPES.INT);
- }
-
- @Test
- public void testSmallInt() {
- dataTypeTest(DATATYPES.SMALLINT);
- }
-
- @Test
- public void testSmallInt1() {
- }
-
- @Test
- public void testSmallInt2() {
- }
-
- @Test
- public void testTinyint() {
- dataTypeTest(DATATYPES.TINYINT);
-
- }
-
- @Test
- public void testTinyInt1() {
- }
-
- @Test
- public void testTinyInt2() {
- }
-
- @Test
- public void testFloat() {
- dataTypeTest(DATATYPES.FLOAT);
- }
-
- @Test
- public void testReal() {
- dataTypeTest(DATATYPES.REAL);
- }
-
- @Test
- public void testDate() {
- dataTypeTest(DATATYPES.DATE);
- }
-
- @Test
- public void testMoney() {
- dataTypeTest(DATATYPES.MONEY);
- }
-
- @Test
- public void testSmallMoney() {
- dataTypeTest(DATATYPES.SMALLMONEY);
- }
-
- @Test
- public void testText() {
- dataTypeTest(DATATYPES.TEXT);
- }
-
- @Test
- public void testVarchar() {
- dataTypeTest(DATATYPES.VARCHAR);
- }
-
- @Test
- public void testChar() {
- dataTypeTest(DATATYPES.CHAR);
- }
-
- @Test
- public void testNText() {
- dataTypeTest(DATATYPES.NTEXT);
- }
-
- @Test
- public void testNChar() {
- dataTypeTest(DATATYPES.NCHAR);
- }
-
- @Test
- public void testNVarchar() {
- dataTypeTest(DATATYPES.NVARCHAR);
- }
-
- @Test
- public void testImage() {
- dataTypeTest(DATATYPES.IMAGE);
- }
-
- @Test
- public void testBinary() {
- dataTypeTest(DATATYPES.BINARY);
- }
-
- //---------------disabled tests-----
- @Test
- public void testTime1() {
- }
-
- @Test
- public void testTime2() {
- }
-
- @Test
- public void testTime3() {
- }
-
- @Test
- public void testTime4() {
- }
-
- @Test
- public void testStringCol1() {
-
- }
-
- @Test
- public void testStringCol2() {
-
- }
-
- @Test
- public void testEmptyStringCol() {
-
- }
-
- @Test
- public void testNullStringCol() {
-
- }
-
- @Test
- public void testNullInt() {
-
- }
-
- @Test
- public void testReal1() {
-
- }
-
- @Test
- public void testReal2() {
-
- }
-
- @Test
- public void testFloat1() {
-
- }
-
- @Test
- public void testFloat2() {
-
- }
-
- @Test
- public void testDate1() {
-
- }
-
- @Test
- public void testDate2() {
-
- }
-
-
- @Test
- public void testNumeric3() {
-
- }
-
- @Test
- public void testNumeric4() {
-
- }
-
- @Test
- public void testNumeric5() {
-
-
- }
-
- @Test
- public void testNumeric6() {
-
- }
-
-
-
- @Test
- public void testDecimal3() {
-
- }
-
- @Test
- public void testDecimal4() {
-
- }
-
- @Test
- public void testDecimal5() {
-
-
- }
-
- @Test
- public void testDecimal6() {
-
- }
-
-
-
- //end disabled tests----------------------------
-
- public String getTrueBoolDbOutput() {
- return "1";
- }
-
- public String getFalseBoolDbOutput() {
- return "0";
- }
-
- protected String getFalseBoolSeqOutput() {
- return "false";
- }
-
- protected String getFalseBoolLiteralSqlInput() {
- return "0";
- }
-
- protected String getFixedCharSeqOut(int len, String val) {
- return val + nSpace(len - val.length());
- }
-
- protected String getFixedCharDbOut(int len, String val) {
- return val + nSpace(len - val.length());
- }
-
- public String nSpace(int n) {
- String tmp = "";
- for (int i = 0; i < n; i++) {
- tmp += " ";
- }
-
- return tmp;
- }
-
- public String nZeros(int n) {
- String tmp = "";
- for (int i = 0; i < n; i++) {
- tmp += "0";
- }
-
- return tmp;
- }
-
- public void dataTypeTest(DATATYPES datatype) {
- int exceptionCount = 0;
-
- List testdata = tdfs.getTestdata(datatype);
-
- for (Iterator<MSSQLTestData> itr = testdata.iterator(); itr.hasNext();) {
- MSSQLTestData current = itr.next();
- System.out.println("Testing with : \n" + current);
-
- try {
- if (datatype == DATATYPES.DECIMAL
- || datatype == DATATYPES.NUMERIC) {
-
- verifyType(current.getDatatype() + "("
- + current.getData(KEY_STRINGS.SCALE) + ","
- + current.getData(KEY_STRINGS.PREC) + ")", current
- .getData(KEY_STRINGS.TO_INSERT), current
- .getData(KEY_STRINGS.HDFS_READBACK));
-
- } else if (datatype == DATATYPES.TIME
- || datatype == DATATYPES.SMALLDATETIME
- || datatype == DATATYPES.DATETIME2
- || datatype == DATATYPES.DATETIME
- || datatype == DATATYPES.DATETIMEOFFSET
- || datatype == DATATYPES.TEXT
- || datatype == DATATYPES.NTEXT
- || datatype == DATATYPES.DATE) {
- verifyType(current.getDatatype(), "'"
- + current.getData(KEY_STRINGS.TO_INSERT) + "'", current
- .getData(KEY_STRINGS.HDFS_READBACK));
- } else if (datatype == DATATYPES.VARBINARY) {
- verifyType(
- current.getDatatype() + "("
- + current.getData(KEY_STRINGS.SCALE) + ")",
- "cast('" + current.getData(KEY_STRINGS.TO_INSERT)
- + "' as varbinary("
- + current.getData(KEY_STRINGS.SCALE) + "))",
- current.getData(KEY_STRINGS.HDFS_READBACK));
- } else if (datatype == DATATYPES.BINARY) {
- verifyType(
- current.getDatatype() + "("
- + current.getData(KEY_STRINGS.SCALE) + ")",
- "cast('" + current.getData(KEY_STRINGS.TO_INSERT)
- + "' as binary("
- + current.getData(KEY_STRINGS.SCALE) + "))",
- current.getData(KEY_STRINGS.HDFS_READBACK));
- } else if (datatype == DATATYPES.NCHAR
- || datatype == DATATYPES.VARCHAR
- || datatype == DATATYPES.CHAR
- || datatype == DATATYPES.NVARCHAR) {
- System.out.println("------>"
- + current.getData(KEY_STRINGS.DB_READBACK)
- + "<----");
- verifyType(current.getDatatype() + "("
- + current.getData(KEY_STRINGS.SCALE) + ")", "'"
- + current.getData(KEY_STRINGS.TO_INSERT) + "'",
- current.getData(KEY_STRINGS.HDFS_READBACK));
- } else if (datatype == DATATYPES.IMAGE) {
- verifyType(current.getDatatype(), "cast('"
- + current.getData(KEY_STRINGS.TO_INSERT)
- + "' as image )",
- current.getData(KEY_STRINGS.HDFS_READBACK));
- } else {
- verifyType(current.getDatatype(), current
- .getData(KEY_STRINGS.TO_INSERT), current
- .getData(KEY_STRINGS.HDFS_READBACK));
- }
-
- addToReport(current, null);
-
- } catch (AssertionError ae) {
- if (current.getData(KEY_STRINGS.NEG_POS_FLAG).equals("NEG")) {
- System.out.println("failure was expected, PASS");
- addToReport(current, null);
- } else {
- System.out
- .println("----------------------------------------------------------"
- + "-");
- System.out.println("Failure for following Test Data :\n"
- + current.toString());
- System.out
- .println("----------------------------------------------------------"
- + "-");
- System.out.println("Exception details : \n");
- System.out.println(ae.getMessage());
- System.out
- .println("----------------------------------------------------------"
- + "-");
- addToReport(current, ae);
- exceptionCount++;
- }
- } catch (Exception e) {
- addToReport(current, e);
- exceptionCount++;
- }
- }
-
- if (exceptionCount > 0) {
- System.out.println("There were failures for :"
- + datatype.toString());
- System.out.println("Failed for " + exceptionCount
- + " test data samples\n");
- System.out.println("Sroll up for detailed errors");
- System.out
- .println("-----------------------------------------------------------");
- throw new AssertionError("Failed for " + exceptionCount
- + " test data sample");
- }
- }
-
- public synchronized void addToReport(MSSQLTestData td, Object result) {
- System.out.println("called");
- try {
- FileWriter fr = new FileWriter(getResportFileName(), true);
- String offset = td.getData(KEY_STRINGS.OFFSET);
- String res = "_";
- if (result == null) {
- res = "Success";
- } else {
- try {
- res = "FAILED "
- + removeNewLines(((AssertionError) result)
- .getMessage());
- } catch (Exception ae) {
- if (result instanceof Exception
- && ((Exception) result) != null) {
- res = "FAILED "
- + removeNewLines(((Exception) result)
- .getMessage());
- } else {
- res = "FAILED " + result.toString();
- }
- }
- }
-
- fr.append(offset + "\t" + res + "\n");
- fr.close();
- } catch (Exception e) {
- LOG.error(StringUtils.stringifyException(e));
- }
- }
-
- public static String removeNewLines(String str) {
- String[] tmp = str.split("\n");
- String result = "";
- for (String a : tmp) {
- result += " " + a;
- }
- return result;
- }
-
- public String getResportFileName(){
- return this.getClass().toString()+".txt";
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileTest.java
new file mode 100644
index 0000000..409c4ad
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerDatatypeImportSequenceFileTest.java
@@ -0,0 +1,847 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.manager.sqlserver;
+
+import java.io.FileWriter;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.testutil.ManagerCompatTestCase;
+import org.apache.sqoop.manager.sqlserver.MSSQLTestDataFileParser.DATATYPES;
+import org.apache.sqoop.manager.sqlserver.MSSQLTestData.KEY_STRINGS;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test importing sequence file from SQL Server.
+ *
+ * This uses JDBC to import data from an SQLServer database to HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerDatatypeImportSequenceFileTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerDatatypeImportSequenceFileTest extends
+ ManagerCompatTestCase {
+
+ public static final Log LOG = LogFactory.getLog(
+ SQLServerDatatypeImportSequenceFileTest.class.getName());
+ private static MSSQLTestDataFileParser tdfs;
+ private static Map report;
+
+ static {
+ try {
+
+ String testfile = null;
+ testfile = System.getProperty("test.data.dir")
+ + "/" + System.getProperty("ms.datatype.test.data.file.import");
+ String delim = System.getProperty("ms.datatype.test.data.file.delim", ",");
+ System.out.println("Using data file : " + testfile);
+ LOG.info("Using data file : " + testfile);
+ tdfs = new MSSQLTestDataFileParser(testfile);
+ tdfs.setDelim(delim);
+ tdfs.parse();
+ report = new HashMap();
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ System.out
+ .println("Error with test data file, check stack trace for cause"
+ + ".\nTests cannont continue.");
+ System.exit(0);
+ }
+ }
+
+ @Override
+ protected String getDbFriendlyName() {
+ return "MSSQL";
+ }
+
+ @Override
+ protected Log getLogger() {
+ return LOG;
+ }
+
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ protected String getConnectString() {
+ return MSSQLTestUtils.getDBConnectString();
+ }
+
+ /**
+ * Drop a table if it already exists in the database.
+ *
+ * @param table
+ * the name of the table to drop.
+ * @throws SQLException
+ * if something goes wrong.
+ */
+ protected void dropTableIfExists(String table) throws SQLException {
+ Connection conn = getManager().getConnection();
+ String sqlStmt = "IF OBJECT_ID('" + table
+ + "') IS NOT NULL DROP TABLE " + table;
+
+ PreparedStatement statement = conn.prepareStatement(sqlStmt,
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ statement.executeUpdate();
+ conn.commit();
+ } finally {
+ statement.close();
+ }
+ }
+
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ SqoopOptions opts = new SqoopOptions(conf);
+ opts.setUsername(username);
+ opts.setPassword(password);
+ return opts;
+ }
+
+ @Before
+ public void setUp() {
+ try {
+ super.setUp();
+ } catch (Exception e) {
+ try {
+ FileWriter fr = new FileWriter(getResportFileName(), true);
+ String res = removeNewLines(e.getMessage());
+ fr.append("Error\t" + res + "\n");
+ fr.close();
+ } catch (Exception e2) {
+ LOG.error(StringUtils.stringifyException(e2));
+ fail(e2.toString());
+ }
+ } catch (Error e) {
+ try {
+ FileWriter fr = new FileWriter(getResportFileName(), true);
+
+ String res = removeNewLines(e.getMessage());
+
+ fr.append("Error\t" + res + "\n");
+ fr.close();
+ fail(res);
+ } catch (Exception e2) {
+ LOG.error(StringUtils.stringifyException(e2));
+ fail(e2.toString());
+ }
+ }
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ super.tearDown();
+ } catch (Exception e) {
+ try {
+ FileWriter fr = new FileWriter(getResportFileName(), true);
+ String res = removeNewLines(e.getMessage());
+ fr.append("Error\t" + res + "\n");
+ fr.close();
+ } catch (Exception e2) {
+ LOG.error(StringUtils.stringifyException(e2));
+ fail(e2.toString());
+ }
+ } catch (Error e) {
+ try {
+ FileWriter fr = new FileWriter(getResportFileName(), true);
+ String res = removeNewLines(e.getMessage());
+ fr.append("Error\t" + res + "\n");
+ fr.close();
+ fail(res);
+ } catch (Exception e2) {
+ LOG.error(StringUtils.stringifyException(e2));
+ fail(e2.toString());
+ }
+ }
+ }
+
+ protected boolean supportsBoolean() {
+ return true;
+ }
+
+ @Test
+ public void testBit() {
+ if (!supportsBoolean()) {
+ skipped = true;
+ return;
+ }
+ verifyType("BIT", getTrueBoolNumericSqlInput(), getTrueBoolSeqOutput());
+ }
+
+ @Test
+ public void testBit2() {
+ if (!supportsBoolean()) {
+ skipped = true;
+ return;
+ }
+ verifyType("BIT", getFalseBoolNumericSqlInput(), getFalseBoolSeqOutput());
+ }
+
+ @Test
+ public void testBit3() {
+ if (!supportsBoolean()) {
+ skipped = true;
+ return;
+ }
+ verifyType("BIT", getFalseBoolLiteralSqlInput(), getFalseBoolSeqOutput());
+ }
+
+ @Test
+ public void testBoolean() {
+ try {
+ super.testBoolean();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testBoolean2() {
+ try {
+ super.testBoolean2();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testBoolean3() {
+ try {
+ super.testBoolean3();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testDouble1() {
+ try {
+ super.testDouble1();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testDouble2() {
+ try {
+ super.testDouble2();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testClob1() {
+ try {
+ super.testClob1();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testBlob1() {
+ try {
+ super.testBlob1();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testLongVarChar() {
+ try {
+ super.testLongVarChar();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testTimestamp1() {
+ try {
+ super.testTimestamp1();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testTimestamp2() {
+ try {
+ super.testTimestamp2();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testTimestamp3() {
+ try {
+ super.testTimestamp3();
+ assertTrue("This test should not pass on sql server", false);
+ } catch (AssertionError a) {
+ System.out.println("Test failed, this was expected");
+ }
+ }
+
+ @Test
+ public void testVarBinary() {
+ if (!supportsVarBinary()) {
+ return;
+ }
+ dataTypeTest(DATATYPES.VARBINARY);
+ }
+
+ @Test
+ public void testTime() {
+ if (!supportsTime()) {
+ skipped = true;
+ return;
+ }
+ dataTypeTest(DATATYPES.TIME);
+ }
+
+ @Test
+ public void testSmalldatetime() {
+ if (!supportsTime()) {
+ skipped = true;
+ return;
+ }
+ dataTypeTest(DATATYPES.SMALLDATETIME);
+ }
+
+ @Test
+ public void testdatetime2() {
+ if (!supportsTime()) {
+ skipped = true;
+ return;
+ }
+ dataTypeTest(DATATYPES.DATETIME2);
+ }
+
+ @Test
+ public void testdatetime() {
+ if (!supportsTime()) {
+ skipped = true;
+ return;
+ }
+ dataTypeTest(DATATYPES.DATETIME);
+ }
+
+ @Test
+ public void testdatetimeoffset() {
+ if (!supportsTime()) {
+ skipped = true;
+ return;
+ }
+ dataTypeTest(DATATYPES.DATETIMEOFFSET);
+ }
+
+ @Test
+ public void testDecimal() {
+ dataTypeTest(DATATYPES.DECIMAL);
+ }
+
+ @Test
+ public void testNumeric() {
+ dataTypeTest(DATATYPES.NUMERIC);
+ }
+
+ @Test
+ public void testNumeric1() {
+ }
+
+ @Test
+ public void testNumeric2() {
+ }
+
+ @Test
+ public void testDecimal1() {
+ }
+
+ @Test
+ public void testDecimal2() {
+ }
+
+ @Test
+ public void testBigInt() {
+ dataTypeTest(DATATYPES.BIGINT);
+ }
+
+ @Test
+ public void testBigInt1() {
+ }
+
+ @Test
+ public void testInt() {
+ dataTypeTest(DATATYPES.INT);
+ }
+
+ @Test
+ public void testSmallInt() {
+ dataTypeTest(DATATYPES.SMALLINT);
+ }
+
+ @Test
+ public void testSmallInt1() {
+ }
+
+ @Test
+ public void testSmallInt2() {
+ }
+
+ @Test
+ public void testTinyint() {
+ dataTypeTest(DATATYPES.TINYINT);
+
+ }
+
+ @Test
+ public void testTinyInt1() {
+ }
+
+ @Test
+ public void testTinyInt2() {
+ }
+
+ @Test
+ public void testFloat() {
+ dataTypeTest(DATATYPES.FLOAT);
+ }
+
+ @Test
+ public void testReal() {
+ dataTypeTest(DATATYPES.REAL);
+ }
+
+ @Test
+ public void testDate() {
+ dataTypeTest(DATATYPES.DATE);
+ }
+
+ @Test
+ public void testMoney() {
+ dataTypeTest(DATATYPES.MONEY);
+ }
+
+ @Test
+ public void testSmallMoney() {
+ dataTypeTest(DATATYPES.SMALLMONEY);
+ }
+
+ @Test
+ public void testText() {
+ dataTypeTest(DATATYPES.TEXT);
+ }
+
+ @Test
+ public void testVarchar() {
+ dataTypeTest(DATATYPES.VARCHAR);
+ }
+
+ @Test
+ public void testChar() {
+ dataTypeTest(DATATYPES.CHAR);
+ }
+
+ @Test
+ public void testNText() {
+ dataTypeTest(DATATYPES.NTEXT);
+ }
+
+ @Test
+ public void testNChar() {
+ dataTypeTest(DATATYPES.NCHAR);
+ }
+
+ @Test
+ public void testNVarchar() {
+ dataTypeTest(DATATYPES.NVARCHAR);
+ }
+
+ @Test
+ public void testImage() {
+ dataTypeTest(DATATYPES.IMAGE);
+ }
+
+ @Test
+ public void testBinary() {
+ dataTypeTest(DATATYPES.BINARY);
+ }
+
+ //---------------disabled tests-----
+ @Test
+ public void testTime1() {
+ }
+
+ @Test
+ public void testTime2() {
+ }
+
+ @Test
+ public void testTime3() {
+ }
+
+ @Test
+ public void testTime4() {
+ }
+
+ @Test
+ public void testStringCol1() {
+
+ }
+
+ @Test
+ public void testStringCol2() {
+
+ }
+
+ @Test
+ public void testEmptyStringCol() {
+
+ }
+
+ @Test
+ public void testNullStringCol() {
+
+ }
+
+ @Test
+ public void testNullInt() {
+
+ }
+
+ @Test
+ public void testReal1() {
+
+ }
+
+ @Test
+ public void testReal2() {
+
+ }
+
+ @Test
+ public void testFloat1() {
+
+ }
+
+ @Test
+ public void testFloat2() {
+
+ }
+
+ @Test
+ public void testDate1() {
+
+ }
+
+ @Test
+ public void testDate2() {
+
+ }
+
+
+ @Test
+ public void testNumeric3() {
+
+ }
+
+ @Test
+ public void testNumeric4() {
+
+ }
+
+ @Test
+ public void testNumeric5() {
+
+
+ }
+
+ @Test
+ public void testNumeric6() {
+
+ }
+
+
+
+ @Test
+ public void testDecimal3() {
+
+ }
+
+ @Test
+ public void testDecimal4() {
+
+ }
+
+ @Test
+ public void testDecimal5() {
+
+
+ }
+
+ @Test
+ public void testDecimal6() {
+
+ }
+
+
+
+ //end disabled tests----------------------------
+
+ public String getTrueBoolDbOutput() {
+ return "1";
+ }
+
+ public String getFalseBoolDbOutput() {
+ return "0";
+ }
+
+ protected String getFalseBoolSeqOutput() {
+ return "false";
+ }
+
+ protected String getFalseBoolLiteralSqlInput() {
+ return "0";
+ }
+
+ protected String getFixedCharSeqOut(int len, String val) {
+ return val + nSpace(len - val.length());
+ }
+
+ protected String getFixedCharDbOut(int len, String val) {
+ return val + nSpace(len - val.length());
+ }
+
+ public String nSpace(int n) {
+ String tmp = "";
+ for (int i = 0; i < n; i++) {
+ tmp += " ";
+ }
+
+ return tmp;
+ }
+
+ public String nZeros(int n) {
+ String tmp = "";
+ for (int i = 0; i < n; i++) {
+ tmp += "0";
+ }
+
+ return tmp;
+ }
+
+ public void dataTypeTest(DATATYPES datatype) {
+ int exceptionCount = 0;
+
+ List testdata = tdfs.getTestdata(datatype);
+
+ for (Iterator<MSSQLTestData> itr = testdata.iterator(); itr.hasNext();) {
+ MSSQLTestData current = itr.next();
+ System.out.println("Testing with : \n" + current);
+
+ try {
+ if (datatype == DATATYPES.DECIMAL
+ || datatype == DATATYPES.NUMERIC) {
+
+ verifyType(current.getDatatype() + "("
+ + current.getData(KEY_STRINGS.SCALE) + ","
+ + current.getData(KEY_STRINGS.PREC) + ")", current
+ .getData(KEY_STRINGS.TO_INSERT), current
+ .getData(KEY_STRINGS.HDFS_READBACK));
+
+ } else if (datatype == DATATYPES.TIME
+ || datatype == DATATYPES.SMALLDATETIME
+ || datatype == DATATYPES.DATETIME2
+ || datatype == DATATYPES.DATETIME
+ || datatype == DATATYPES.DATETIMEOFFSET
+ || datatype == DATATYPES.TEXT
+ || datatype == DATATYPES.NTEXT
+ || datatype == DATATYPES.DATE) {
+ verifyType(current.getDatatype(), "'"
+ + current.getData(KEY_STRINGS.TO_INSERT) + "'", current
+ .getData(KEY_STRINGS.HDFS_READBACK));
+ } else if (datatype == DATATYPES.VARBINARY) {
+ verifyType(
+ current.getDatatype() + "("
+ + current.getData(KEY_STRINGS.SCALE) + ")",
+ "cast('" + current.getData(KEY_STRINGS.TO_INSERT)
+ + "' as varbinary("
+ + current.getData(KEY_STRINGS.SCALE) + "))",
+ current.getData(KEY_STRINGS.HDFS_READBACK));
+ } else if (datatype == DATATYPES.BINARY) {
+ verifyType(
+ current.getDatatype() + "("
+ + current.getData(KEY_STRINGS.SCALE) + ")",
+ "cast('" + current.getData(KEY_STRINGS.TO_INSERT)
+ + "' as binary("
+ + current.getData(KEY_STRINGS.SCALE) + "))",
+ current.getData(KEY_STRINGS.HDFS_READBACK));
+ } else if (datatype == DATATYPES.NCHAR
+ || datatype == DATATYPES.VARCHAR
+ || datatype == DATATYPES.CHAR
+ || datatype == DATATYPES.NVARCHAR) {
+ System.out.println("------>"
+ + current.getData(KEY_STRINGS.DB_READBACK)
+ + "<----");
+ verifyType(current.getDatatype() + "("
+ + current.getData(KEY_STRINGS.SCALE) + ")", "'"
+ + current.getData(KEY_STRINGS.TO_INSERT) + "'",
+ current.getData(KEY_STRINGS.HDFS_READBACK));
+ } else if (datatype == DATATYPES.IMAGE) {
+ verifyType(current.getDatatype(), "cast('"
+ + current.getData(KEY_STRINGS.TO_INSERT)
+ + "' as image )",
+ current.getData(KEY_STRINGS.HDFS_READBACK));
+ } else {
+ verifyType(current.getDatatype(), current
+ .getData(KEY_STRINGS.TO_INSERT), current
+ .getData(KEY_STRINGS.HDFS_READBACK));
+ }
+
+ addToReport(current, null);
+
+ } catch (AssertionError ae) {
+ if (current.getData(KEY_STRINGS.NEG_POS_FLAG).equals("NEG")) {
+ System.out.println("failure was expected, PASS");
+ addToReport(current, null);
+ } else {
+ System.out
+ .println("----------------------------------------------------------"
+ + "-");
+ System.out.println("Failure for following Test Data :\n"
+ + current.toString());
+ System.out
+ .println("----------------------------------------------------------"
+ + "-");
+ System.out.println("Exception details : \n");
+ System.out.println(ae.getMessage());
+ System.out
+ .println("----------------------------------------------------------"
+ + "-");
+ addToReport(current, ae);
+ exceptionCount++;
+ }
+ } catch (Exception e) {
+ addToReport(current, e);
+ exceptionCount++;
+ }
+ }
+
+ if (exceptionCount > 0) {
+ System.out.println("There were failures for :"
+ + datatype.toString());
+ System.out.println("Failed for " + exceptionCount
+ + " test data samples\n");
+ System.out.println("Sroll up for detailed errors");
+ System.out
+ .println("-----------------------------------------------------------");
+ throw new AssertionError("Failed for " + exceptionCount
+ + " test data sample");
+ }
+ }
+
+ public synchronized void addToReport(MSSQLTestData td, Object result) {
+ System.out.println("called");
+ try {
+ FileWriter fr = new FileWriter(getResportFileName(), true);
+ String offset = td.getData(KEY_STRINGS.OFFSET);
+ String res = "_";
+ if (result == null) {
+ res = "Success";
+ } else {
+ try {
+ res = "FAILED "
+ + removeNewLines(((AssertionError) result)
+ .getMessage());
+ } catch (Exception ae) {
+ if (result instanceof Exception
+ && ((Exception) result) != null) {
+ res = "FAILED "
+ + removeNewLines(((Exception) result)
+ .getMessage());
+ } else {
+ res = "FAILED " + result.toString();
+ }
+ }
+ }
+
+ fr.append(offset + "\t" + res + "\n");
+ fr.close();
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ }
+ }
+
+ public static String removeNewLines(String str) {
+ String[] tmp = str.split("\n");
+ String result = "";
+ for (String a : tmp) {
+ result += " " + a;
+ }
+ return result;
+ }
+
+ public String getResportFileName(){
+ return this.getClass().toString()+".txt";
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java
deleted file mode 100644
index ab3dd08..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportManualTest.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.manager.sqlserver;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.StringUtils;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.hive.TestHiveImport;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.tool.SqoopTool;
-import org.junit.After;
-import org.junit.Before;
-
-import static org.junit.Assert.fail;
-
-/**
- * Test import to Hive from SQL Server.
- *
- * This uses JDBC to import data from an SQLServer database to HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerHiveImportManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerHiveImportManualTest extends TestHiveImport {
-
- @Before
- public void setUp() {
- super.setUp();
- }
-
- @After
- public void tearDown() {
- try {
- dropTableIfExists(getTableName());
- } catch (SQLException sqle) {
- LOG.info("Table clean-up failed: " + sqle);
- } finally {
- super.tearDown();
- }
- }
-
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- protected String getConnectString() {
- return MSSQLTestUtils.getDBConnectString();
- }
-
- //SQL Server pads out
- @Override
- protected String[] getTypes() {
- String[] types = { "VARCHAR(32)", "INTEGER", "VARCHAR(64)" };
- return types;
- }
-
- /**
- * Drop a table if it already exists in the database.
- * @param table
- * the name of the table to drop.
- * @throws SQLException
- * if something goes wrong.
- */
- protected void dropTableIfExists(String table) throws SQLException {
- Connection conn = getManager().getConnection();
- String sqlStmt = "IF OBJECT_ID('" + table
- + "') IS NOT NULL DROP TABLE " + table;
- PreparedStatement statement = conn.prepareStatement(sqlStmt,
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- conn.commit();
- } finally {
- statement.close();
- }
- }
-
- protected SqoopOptions getSqoopOptions(Configuration conf) {
-
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- SqoopOptions opts = new SqoopOptions(conf);
- opts.setUsername(username);
- opts.setPassword(password);
-
- return opts;
-
- }
-
- SqoopOptions getSqoopOptions(String[] args, SqoopTool tool) {
- SqoopOptions opts = null;
- try {
- opts = tool.parseArguments(args, null, null, true);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- opts.setUsername(username);
- opts.setPassword(password);
-
- } catch (Exception e) {
- LOG.error(StringUtils.stringifyException(e));
- fail("Invalid options: " + e.toString());
- }
-
- return opts;
- }
-
- protected String[] getArgv(boolean includeHadoopFlags, String[] moreArgs) {
- ArrayList<String> args = new ArrayList<String>();
- System.out.println("Overridden getArgv is called..");
- if (includeHadoopFlags) {
- CommonArgs.addHadoopFlags(args);
- }
-
- if (null != moreArgs) {
- for (String arg : moreArgs) {
- args.add(arg);
- }
- }
-
- args.add("--table");
- args.add(getTableName());
- args.add("--warehouse-dir");
- args.add(getWarehouseDir());
- args.add("--connect");
- args.add(getConnectString());
- args.add("--hive-import");
- String[] colNames = getColNames();
- if (null != colNames) {
- args.add("--split-by");
- args.add(colNames[0]);
- } else {
- fail("Could not determine column names.");
- }
-
- args.add("--num-mappers");
- args.add("1");
-
- for (String a : args) {
- LOG.debug("ARG : " + a);
- }
-
- return args.toArray(new String[0]);
- }
-
- protected String[] getCodeGenArgs() {
- ArrayList<String> args = new ArrayList<String>();
-
- args.add("--table");
- args.add(getTableName());
- args.add("--connect");
- args.add(getConnectString());
- args.add("--hive-import");
-
- return args.toArray(new String[0]);
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportTest.java
new file mode 100644
index 0000000..535e599
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerHiveImportTest.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.manager.sqlserver;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.hive.TestHiveImport;
+import com.cloudera.sqoop.testutil.CommonArgs;
+import com.cloudera.sqoop.tool.SqoopTool;
+import org.junit.After;
+import org.junit.Before;
+
+import static org.junit.Assert.fail;
+
+/**
+ * Test import to Hive from SQL Server.
+ *
+ * This uses JDBC to import data from an SQLServer database to HDFS.
+ *
+ * Since this requires an SQLServer installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=SQLServerHiveImportTest or -Dthirdparty=true.
+ *
+ * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons) and set it's path through -Dsqoop.thirdparty.lib.dir.
+ *
+ * To set up your test environment:
+ * Install SQL Server Express 2012
+ * Create a database SQOOPTEST
+ * Create a login SQOOPUSER with password PASSWORD and grant all
+ * access for SQOOPTEST to SQOOPUSER.
+ * Set these through -Dsqoop.test.sqlserver.connectstring.host_url, -Dsqoop.test.sqlserver.database and
+ * -Dms.sqlserver.password
+ */
+public class SQLServerHiveImportTest extends TestHiveImport {
+
+ @Before
+ public void setUp() {
+ super.setUp();
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ dropTableIfExists(getTableName());
+ } catch (SQLException sqle) {
+ LOG.info("Table clean-up failed: " + sqle);
+ } finally {
+ super.tearDown();
+ }
+ }
+
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ protected String getConnectString() {
+ return MSSQLTestUtils.getDBConnectString();
+ }
+
+ //SQL Server pads out
+ @Override
+ protected String[] getTypes() {
+ String[] types = { "VARCHAR(32)", "INTEGER", "VARCHAR(64)" };
+ return types;
+ }
+
+ /**
+ * Drop a table if it already exists in the database.
+ * @param table
+ * the name of the table to drop.
+ * @throws SQLException
+ * if something goes wrong.
+ */
+ protected void dropTableIfExists(String table) throws SQLException {
+ Connection conn = getManager().getConnection();
+ String sqlStmt = "IF OBJECT_ID('" + table
+ + "') IS NOT NULL DROP TABLE " + table;
+ PreparedStatement statement = conn.prepareStatement(sqlStmt,
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ statement.executeUpdate();
+ conn.commit();
+ } finally {
+ statement.close();
+ }
+ }
+
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ SqoopOptions opts = new SqoopOptions(conf);
+ opts.setUsername(username);
+ opts.setPassword(password);
+
+ return opts;
+
+ }
+
+ SqoopOptions getSqoopOptions(String[] args, SqoopTool tool) {
+ SqoopOptions opts = null;
+ try {
+ opts = tool.parseArguments(args, null, null, true);
+ String username = MSSQLTestUtils.getDBUserName();
+ String password = MSSQLTestUtils.getDBPassWord();
+ opts.setUsername(username);
+ opts.setPassword(password);
+
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ fail("Invalid options: " + e.toString());
+ }
+
+ return opts;
+ }
+
+ protected String[] getArgv(boolean includeHadoopFlags, String[] moreArgs) {
+ ArrayList<String> args = new ArrayList<String>();
+ System.out.println("Overridden getArgv is called..");
+ if (includeHadoopFlags) {
+ CommonArgs.addHadoopFlags(args);
+ }
+
+ if (null != moreArgs) {
+ for (String arg : moreArgs) {
+ args.add(arg);
+ }
+ }
+
+ args.add("--table");
+ args.add(getTableName());
+ args.add("--warehouse-dir");
+ args.add(getWarehouseDir());
+ args.add("--connect");
+ args.add(getConnectString());
+ args.add("--hive-import");
+ String[] colNames = getColNames();
+ if (null != colNames) {
+ args.add("--split-by");
+ args.add(colNames[0]);
+ } else {
+ fail("Could not determine column names.");
+ }
+
+ args.add("--num-mappers");
+ args.add("1");
+
+ for (String a : args) {
+ LOG.debug("ARG : " + a);
+ }
+
+ return args.toArray(new String[0]);
+ }
+
+ protected String[] getCodeGenArgs() {
+ ArrayList<String> args = new ArrayList<String>();
+
+ args.add("--table");
+ args.add(getTableName());
+ args.add("--connect");
+ args.add(getConnectString());
+ args.add("--hive-import");
+
+ return args.toArray(new String[0]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/558bdaea/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java
deleted file mode 100644
index 8cc5a0b..0000000
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerManualTest.java
+++ /dev/null
@@ -1,366 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sqoop.manager.sqlserver;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Types;
-import java.util.Map;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.StringUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import com.cloudera.sqoop.ConnFactory;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.manager.ConnManager;
-import com.cloudera.sqoop.metastore.JobData;
-import com.cloudera.sqoop.testutil.HsqldbTestServer;
-import com.cloudera.sqoop.tool.ImportTool;
-import com.cloudera.sqoop.tool.SqoopTool;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
-
-/**
- * Test methods of the generic SqlManager implementation.
- *
- * This uses JDBC to import data from an SQLServer database to HDFS.
- *
- * Since this requires an SQLServer installation,
- * this class is named in such a way that Sqoop's default QA process does
- * not run it. You need to run this manually with
- * -Dtestcase=SQLServerManagerManualTest.
- *
- * You need to put SQL Server JDBC driver library (sqljdbc4.jar) in a location
- * where Sqoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install SQL Server Express 2012
- * Create a database SQOOPTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * access for SQOOPTEST to SQOOPUSER.
- */
-public class SQLServerManagerManualTest {
-
- public static final Log LOG = LogFactory.getLog(
- SQLServerManagerManualTest.class.getName());
-
- /** the name of a table that doesn't exist. */
- static final String MISSING_TABLE = "MISSING_TABLE";
-
- // instance variables populated during setUp, used during tests
- private HsqldbTestServer testServer;
- private ConnManager manager;
-
- @Before
- public void setUp() {
- MSSQLTestUtils utils = new MSSQLTestUtils();
- try {
- utils.createTableFromSQL(MSSQLTestUtils.CREATE_TALBE_LINEITEM);
- utils.populateLineItem();
- } catch (SQLException e) {
- LOG.error("Setup fail with SQLException: " + StringUtils.stringifyException(e));
- fail("Setup fail with SQLException: " + e.toString());
- }
- Configuration conf = getConf();
- SqoopOptions opts = getSqoopOptions(conf);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- opts.setUsername(username);
- opts.setPassword(password);
- opts.setConnectString(getConnectString());
- ConnFactory f = new ConnFactory(conf);
- try {
- this.manager = f.getManager(new JobData(opts, new ImportTool()));
- System.out.println("Manger : " + this.manager);
- } catch (IOException ioe) {
- LOG.error("Setup fail with IOException: " + StringUtils.stringifyException(ioe));
- fail("IOException instantiating manager: "
- + StringUtils.stringifyException(ioe));
- }
- }
-
- @After
- public void tearDown() {
- try {
-
- MSSQLTestUtils utils = new MSSQLTestUtils();
- utils.dropTableIfExists("TPCH1M_LINEITEM");
- manager.close();
- } catch (SQLException sqlE) {
- LOG.error("Got SQLException: " + sqlE.toString());
- fail("Got SQLException: " + sqlE.toString());
- }
- }
-
- @Test
- public void testListColNames() {
- String[] colNames = manager.getColumnNames(getTableName());
- assertNotNull("manager returned no colname list", colNames);
- assertEquals("Table list should be length 2", 16, colNames.length);
- String[] knownFields = MSSQLTestUtils.getColumns();
- for (int i = 0; i < colNames.length; i++) {
- assertEquals(knownFields[i], colNames[i]);
- }
- }
-
- @Test
- public void testListColTypes() {
- Map<String, Integer> types = manager.getColumnTypes(getTableName());
-
- assertNotNull("manager returned no types map", types);
- assertEquals("Map should be size=16", 16, types.size());
- assertEquals(types.get("L_ORDERKEY").intValue(), Types.INTEGER);
- assertEquals(types.get("L_COMMENT").intValue(), Types.VARCHAR);
- }
-
- @Test
- public void testMissingTableColNames() {
- // SQL Server returns an empty column list which gets translated as a
- // zero length array
- // how ever also check in case it returns null, which is also correct
- String[] colNames = manager.getColumnNames(MISSING_TABLE);
- if (colNames == null) {
- assertNull("No column names should be returned for missing table",
- colNames);
- }
- int numItems = colNames.length;
- assertEquals(0, numItems);
- }
-
- @Test
- public void testMissingTableColTypes() {
- Map<String, Integer> colTypes = manager.getColumnTypes(MISSING_TABLE);
- assertNull("No column types should be returned for missing table",
- colTypes);
- }
-
- // constants related to testReadTable()
- static final int EXPECTED_NUM_ROWS = 4;
- static final int EXPECTED_COL1_SUM = 10;
- static final int EXPECTED_COL2_SUM = 14;
-
- @Test
- public void testReadTable() {
- ResultSet results = null;
- try {
- results = manager.readTable(getTableName(), MSSQLTestUtils
- .getColumns());
-
- assertNotNull("ResultSet from readTable() is null!", results);
-
- ResultSetMetaData metaData = results.getMetaData();
- assertNotNull("ResultSetMetadata is null in readTable()", metaData);
-
- // ensure that we get the correct number of columns back
- assertEquals("Number of returned columns was unexpected!", metaData
- .getColumnCount(), 16);
-
- // should get back 4 rows. They are:
- // 1 2
- // 3 4
- // 5 6
- // 7 8
- // .. so while order isn't guaranteed, we should get back 16 on the
- // left
- // and 20 on the right.
- int sumCol1 = 0, sumCol2 = 0, rowCount = 0;
- while (results.next()) {
- rowCount++;
- sumCol1 += results.getInt(1);
- sumCol2 += results.getInt(2);
- }
-
- assertEquals("Expected 4 rows back", EXPECTED_NUM_ROWS, rowCount);
- assertEquals("Expected left sum of 10", EXPECTED_COL1_SUM, sumCol1);
- assertEquals("Expected right sum of 14", EXPECTED_COL2_SUM, sumCol2);
- } catch (SQLException sqlException) {
- LOG.error(StringUtils.stringifyException(sqlException));
- fail("SQL Exception: " + sqlException.toString());
- } finally {
- if (null != results) {
- try {
- results.close();
- } catch (SQLException sqlE) {
- LOG.error(StringUtils.stringifyException(sqlE));
- fail("SQL Exception in ResultSet.close(): "
- + sqlE.toString());
- }
- }
-
- manager.release();
- }
- }
-
- @Test
- public void testReadMissingTable() {
- ResultSet results = null;
- try {
- String[] colNames = { "*" };
- results = manager.readTable(MISSING_TABLE, colNames);
- assertNull("Expected null resultset from readTable(MISSING_TABLE)",
- results);
- } catch (SQLException sqlException) {
- // we actually expect this pass.
- } finally {
- if (null != results) {
- try {
- results.close();
- } catch (SQLException sqlE) {
- fail("SQL Exception in ResultSet.close(): "
- + sqlE.toString());
- }
- }
-
- manager.release();
- }
- }
-
- @Test
- public void testgetPrimaryKeyFromMissingTable() {
- String primaryKey = manager.getPrimaryKey(MISSING_TABLE);
- assertNull("Expected null pkey for missing table", primaryKey);
- }
-
- @Test
- public void testgetPrimaryKeyFromTableWithoutKey() {
- String primaryKey = manager.getPrimaryKey(getTableName());
- assertNull("Expected null pkey for table without key", primaryKey);
- }
-
- // constants for getPrimaryKeyFromTable()
- static final String TABLE_WITH_KEY = "TABLE_WITH_KEY";
- static final String KEY_FIELD_NAME = "KEYFIELD";
-
- @Test
- public void testgetPrimaryKeyFromTable() {
- // first, create a table with a primary key
- Connection conn = null;
- try {
- conn = getManager().getConnection();
- dropTableIfExists(TABLE_WITH_KEY);
- PreparedStatement statement = conn.prepareStatement("CREATE TABLE "
- + TABLE_WITH_KEY + "(" + KEY_FIELD_NAME
- + " INT NOT NULL PRIMARY KEY, foo INT)",
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- statement.executeUpdate();
- statement.close();
-
- String primaryKey = getManager().getPrimaryKey(TABLE_WITH_KEY);
- assertEquals("Expected null pkey for table without key",
- primaryKey, KEY_FIELD_NAME);
-
- } catch (SQLException sqlException) {
- LOG.error(StringUtils.stringifyException(sqlException));
- fail("Could not create table with primary key: "
- + sqlException.toString());
- } finally {
- if (null != conn) {
- try {
- conn.close();
- } catch (SQLException sqlE) {
- LOG.warn("Got SQLException during close: "
- + sqlE.toString());
- }
- }
- }
-
- }
-
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- protected String getConnectString() {
- return MSSQLTestUtils.getDBConnectString();
- }
-
- /**
- * Drop a table if it already exists in the database.
- *
- * @param table
- * the name of the table to drop.
- * @throws SQLException
- * if something goes wrong.
- */
- protected void dropTableIfExists(String table) throws SQLException {
- Connection conn = getManager().getConnection();
- String sqlStmt = "IF OBJECT_ID('" + table
- + "') IS NOT NULL DROP TABLE " + table;
- PreparedStatement statement = conn.prepareStatement(sqlStmt,
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- conn.commit();
- } finally {
- statement.close();
- }
- }
-
- protected SqoopOptions getSqoopOptions(Configuration conf) {
- SqoopOptions opt = new SqoopOptions(conf);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- SqoopOptions opts = new SqoopOptions(conf);
- opts.setUsername(username);
- opts.setPassword(password);
-
- return opt;
- }
-
- SqoopOptions getSqoopOptions(String[] args, SqoopTool tool) {
- SqoopOptions opts = null;
- try {
- opts = tool.parseArguments(args, null, null, true);
- String username = MSSQLTestUtils.getDBUserName();
- String password = MSSQLTestUtils.getDBPassWord();
- opts.setUsername(username);
- opts.setPassword(password);
-
- } catch (Exception e) {
- LOG.error(StringUtils.stringifyException(e));
- fail("Invalid options: " + e.toString());
- }
-
- return opts;
- }
-
- protected String getTableName() {
- return "tpch1m_lineitem";
- }
-
- protected ConnManager getManager() {
- return manager;
- }
-
- protected Configuration getConf() {
- return new Configuration();
- }
-
-}