You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by an...@apache.org on 2018/01/18 14:03:52 UTC
[06/32] sqoop git commit: SQOOP-3273: Removing com.cloudera.sqoop
packages
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/mysql/MySQLAuthTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/mysql/MySQLAuthTest.java b/src/test/org/apache/sqoop/manager/mysql/MySQLAuthTest.java
new file mode 100644
index 0000000..1e2f70d
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/mysql/MySQLAuthTest.java
@@ -0,0 +1,277 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.mysql;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.FileInputStream;
+import java.io.File;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.sqoop.manager.DirectMySQLManager;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.testutil.CommonArgs;
+import org.apache.sqoop.testutil.ImportJobTestCase;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test authentication and remote access to direct mysqldump-based imports.
+ *
+ * Since this requires a MySQL installation with a properly configured database and user, this
+ * class is named in such a way that Hadoop's default QA process does not run
+ * it. You need to run this manually with -Dtestcase=MySQLAuthTest
+ *
+ * You need to put MySQL's Connector/J JDBC driver library into a location
+ * where Hadoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons).
+ *
+ * If you don't have a database and a user which can be used by Sqoop, you can create them using
+ * the following MySQL commands:
+ *
+ * CREATE DATABASE sqooppasstest;
+ * use mysql;
+ * GRANT ALL PRIVILEGES on sqooppasstest.* TO 'sqooptest'@'localhost'
+ * IDENTIFIED BY '12345';
+ * flush privileges;
+ *
+ * <br/>
+ *
+ * Ant command for running this test case: <br/>
+ * ant clean test
+ * -Dsqoop.thirdparty.lib.dir=mysql_driver_dir
+ * -Dsqoop.test.mysql.connectstring.host_url=jdbc:mysql://mysql_server_address/
+ * -Dsqoop.test.mysql.username=sqooptest
+ * -Dsqoop.test.mysql.password=12345
+ * -Dsqoop.test.mysql.databasename=sqooppasstest
+ * -Dtestcase=MySQLAuthTest
+ *
+ */
+public class MySQLAuthTest extends ImportJobTestCase {
+
+ public static final Log LOG = LogFactory.getLog(
+ MySQLAuthTest.class.getName());
+
+ private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils();
+
+ private List<String> createdTableNames = new ArrayList<>();
+
+ @Override
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ @Before
+ public void setUp() {
+ super.setUp();
+ SqoopOptions options = new SqoopOptions(mySQLTestUtils.getMySqlConnectString(),
+ getTableName());
+ options.setUsername(mySQLTestUtils.getUserName());
+ options.setPassword(mySQLTestUtils.getUserPass());
+
+ LOG.debug("Setting up another MySQLAuthTest: " + mySQLTestUtils.getMySqlConnectString());
+
+ setManager(new DirectMySQLManager(options));
+ }
+
+ @After
+ public void tearDown() {
+ dropAllCreatedTables();
+ super.tearDown();
+ }
+
+ private String [] getArgv(boolean includeHadoopFlags,
+ boolean useDirect, String connectString, String tableName) {
+ ArrayList<String> args = new ArrayList<String>();
+
+ if (includeHadoopFlags) {
+ CommonArgs.addHadoopFlags(args);
+ }
+
+ args.add("--table");
+ args.add(tableName);
+ args.add("--warehouse-dir");
+ args.add(getWarehouseDir());
+ args.add("--connect");
+ args.add(connectString);
+ if (useDirect) {
+ args.add("--direct");
+ }
+ args.add("--username");
+ args.add(mySQLTestUtils.getUserName());
+ args.add("--password");
+ args.add(mySQLTestUtils.getUserPass());
+ args.add("--mysql-delimiters");
+ args.add("--num-mappers");
+ args.add("1");
+
+ return args.toArray(new String[0]);
+ }
+
+ /**
+ * Connect to a db and ensure that password-based authentication
+ * succeeds.
+ */
+ @Test
+ public void testAuthAccess() {
+ createAndPopulateAuthTable();
+ String [] argv = getArgv(true, true, mySQLTestUtils.getMySqlConnectString(), getTableName());
+ try {
+ runImport(argv);
+ } catch (IOException ioe) {
+ LOG.error("Got IOException during import: " + ioe.toString());
+ ioe.printStackTrace();
+ fail(ioe.toString());
+ }
+
+ Path warehousePath = new Path(this.getWarehouseDir());
+ Path tablePath = new Path(warehousePath, getTableName());
+ Path filePath = new Path(tablePath, "part-m-00000");
+
+ File f = new File(filePath.toString());
+ assertTrue("Could not find imported data file", f.exists());
+ BufferedReader r = null;
+ try {
+ // Read through the file and make sure it's all there.
+ r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
+ assertEquals("1,'Aaron'", r.readLine());
+ } catch (IOException ioe) {
+ LOG.error("Got IOException verifying results: " + ioe.toString());
+ ioe.printStackTrace();
+ fail(ioe.toString());
+ } finally {
+ IOUtils.closeStream(r);
+ }
+ }
+
+ @Test
+ public void testZeroTimestamp() throws IOException, SQLException {
+ // MySQL timestamps can hold values whose range causes problems
+ // for java.sql.Timestamp. The MySQLManager adds settings to the
+ // connect string which configure the driver's handling of
+ // zero-valued timestamps. Check that all of these modifications
+ // to the connect string are successful.
+
+ // A connect string with a null 'query' component.
+ doZeroTimestampTest(0, true, mySQLTestUtils.getMySqlConnectString());
+
+ // A connect string with a zero-length query component.
+ doZeroTimestampTest(1, true, mySQLTestUtils.getMySqlConnectString() + "?");
+
+ // A connect string with another argument
+ doZeroTimestampTest(2, true, mySQLTestUtils.getMySqlConnectString() + "?connectTimeout=0");
+ doZeroTimestampTest(3, true, mySQLTestUtils.getMySqlConnectString() + "?connectTimeout=0&");
+
+ // A connect string with the zero-timestamp behavior already
+ // configured.
+ doZeroTimestampTest(4, true, mySQLTestUtils.getMySqlConnectString()
+ + "?zeroDateTimeBehavior=convertToNull");
+
+ // And finally, behavior already configured in such a way as to
+ // cause the timestamp import to fail.
+ doZeroTimestampTest(5, false, mySQLTestUtils.getMySqlConnectString()
+ + "?zeroDateTimeBehavior=exception");
+ }
+
+ public void doZeroTimestampTest(int testNum, boolean expectSuccess,
+ String connectString) throws IOException, SQLException {
+
+ LOG.info("Beginning zero-timestamp test #" + testNum);
+
+ final String tableName = "mysqlTimestampTable" + Integer.toString(testNum);
+
+ createAndPopulateZeroTimestampTable(tableName);
+
+ // Run the import.
+ String [] argv = getArgv(true, false, connectString, tableName);
+ try {
+ runImport(argv);
+ } catch (Exception e) {
+ if (expectSuccess) {
+ // This is unexpected. rethrow.
+ throw new RuntimeException(e);
+ } else {
+ // We expected an error.
+ LOG.info("Got exception running import (expected). msg: " + e);
+ }
+ }
+
+ // Make sure the result file is there.
+ Path warehousePath = new Path(this.getWarehouseDir());
+ Path tablePath = new Path(warehousePath, tableName);
+ Path filePath = new Path(tablePath, "part-m-00000");
+
+ File f = new File(filePath.toString());
+ if (expectSuccess) {
+ assertTrue("Could not find imported data file", f.exists());
+ BufferedReader r = new BufferedReader(new InputStreamReader(
+ new FileInputStream(f)));
+ assertEquals("1,null", r.readLine());
+ IOUtils.closeStream(r);
+ } else {
+ assertFalse("Imported data when expected failure", f.exists());
+ }
+ }
+
+ private void createAndPopulateZeroTimestampTable(String tableName) {
+ String[] colNames = { "id", "ts" };
+ String[] colTypes = { "INT NOT NULL PRIMARY KEY AUTO_INCREMENT", "TIMESTAMP NOT NULL" };
+ String[] colValues = { "NULL", "'0000-00-00 00:00:00.0'" };
+ createTableWithColTypesAndNames(tableName, colNames, colTypes, colValues);
+ createdTableNames.add(tableName);
+ }
+
+ private void dropAllCreatedTables() {
+ try {
+ for (String createdTableName : createdTableNames) {
+ dropTableIfExists(createdTableName);
+ }
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void createAndPopulateAuthTable() {
+ String[] colNames = { "id", "name" };
+ String[] colTypes = { "INT NOT NULL PRIMARY KEY AUTO_INCREMENT", "VARCHAR(24) NOT NULL" };
+ String[] colValues = { "NULL", "'Aaron'" };
+
+ createTableWithColTypesAndNames(colNames, colTypes, colValues);
+ createdTableNames.add(getTableName());
+ }
+
+ protected String dropTableIfExistsCommand(String tableName) {
+ return String.format("DROP TABLE IF EXISTS %s", tableName);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/mysql/MySQLCompatTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/mysql/MySQLCompatTest.java b/src/test/org/apache/sqoop/manager/mysql/MySQLCompatTest.java
new file mode 100644
index 0000000..7e822e6
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/mysql/MySQLCompatTest.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.mysql;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.testutil.ManagerCompatTestCase;
+
+/**
+ * Test the basic mysql connection manager with the various column types.
+ */
+public class MySQLCompatTest extends ManagerCompatTestCase {
+
+ public static final Log LOG = LogFactory.getLog(
+ MySQLCompatTest.class.getName());
+ private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils();
+
+ @Override
+ protected Log getLogger() {
+ return LOG;
+ }
+
+ @Override
+ protected String getDbFriendlyName() {
+ return "MySQL";
+ }
+
+ @Override
+ protected String getConnectString() {
+ return mySQLTestUtils.getMySqlConnectString();
+ }
+
+ @Override
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ SqoopOptions opts = new SqoopOptions(conf);
+ opts.setUsername(mySQLTestUtils.getUserName());
+ mySQLTestUtils.addPasswordIfIsSet(opts);
+ return opts;
+
+ }
+
+ @Override
+ protected void dropTableIfExists(String table) throws SQLException {
+ Connection conn = getManager().getConnection();
+ PreparedStatement statement = conn.prepareStatement(
+ "DROP TABLE IF EXISTS " + table,
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ statement.executeUpdate();
+ conn.commit();
+ } finally {
+ statement.close();
+ }
+ }
+
+ @Override
+ protected String getLongVarCharType() {
+ return "MEDIUMTEXT";
+ }
+
+ @Override
+ protected String getTimestampType() {
+ // return a nullable timestamp type.
+ return "TIMESTAMP NULL";
+ }
+
+ @Override
+ protected String getClobType() {
+ return "MEDIUMTEXT";
+ }
+
+ @Override
+ protected String getBlobType() {
+ return "MEDIUMBLOB";
+ }
+
+ @Override
+ protected String getRealSeqOutput(String realAsInserted) {
+ return withDecimalZero(realAsInserted);
+ }
+
+ @Override
+ protected String getFloatSeqOutput(String floatAsInserted) {
+ return withDecimalZero(floatAsInserted);
+ }
+
+ @Override
+ protected String getDoubleSeqOutput(String doubleAsInserted) {
+ return withDecimalZero(doubleAsInserted);
+ }
+
+ @Override
+ protected String getTimestampSeqOutput(String tsAsInserted) {
+ // We trim timestamps to exactly one tenth of a second.
+ if ("null".equals(tsAsInserted)) {
+ return tsAsInserted;
+ }
+
+ int dotPos = tsAsInserted.indexOf(".");
+ if (-1 == dotPos) {
+ return tsAsInserted + ".0";
+ } else {
+ return tsAsInserted.substring(0, dotPos + 2);
+ }
+ }
+
+ @Override
+ protected String getNumericSeqOutput(String numAsInserted) {
+ // We always pad to exactly the number of digits in
+ // getNumericDecPartDigits().
+
+ int totalDecPartSize = getNumericDecPartDigits();
+ int numPad; // number of digits to pad by.
+
+ int dotPos = numAsInserted.indexOf(".");
+ if (-1 == dotPos) {
+ numAsInserted = numAsInserted + ".";
+ numPad = totalDecPartSize;
+ } else {
+ int existingDecimalSize = numAsInserted.length() - dotPos;
+ numPad = totalDecPartSize - existingDecimalSize;
+ }
+
+ if (numPad < 0) {
+ // We actually have to trim the value.
+ return numAsInserted.substring(0, numAsInserted.length() + numPad + 1);
+ } else {
+ String zeros = "";
+ for (int i = 0; i < numPad; i++) {
+ zeros = zeros + "0";
+ }
+ return numAsInserted + zeros;
+ }
+ }
+
+ @Override
+ protected String getDecimalSeqOutput(String numAsInserted) {
+ return getNumericSeqOutput(numAsInserted);
+ }
+
+ @Test
+ public void testYear() {
+ verifyType("YEAR", "2012", "2012");
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/mysql/MySQLFreeFormQueryTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/mysql/MySQLFreeFormQueryTest.java b/src/test/org/apache/sqoop/manager/mysql/MySQLFreeFormQueryTest.java
new file mode 100644
index 0000000..f4f0b74
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/mysql/MySQLFreeFormQueryTest.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.mysql;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.TestFreeFormQueryImport;
+
+/**
+ * Test free form query import with the MySQL db.
+ */
+public class MySQLFreeFormQueryTest extends TestFreeFormQueryImport {
+
+ public static final Log LOG = LogFactory.getLog(
+ MySQLFreeFormQueryTest.class.getName());
+ private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils();
+
+ @Override
+ protected Log getLogger() {
+ return LOG;
+ }
+
+ @Override
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ @Override
+ protected String getConnectString() {
+ return mySQLTestUtils.getMySqlConnectString();
+ }
+
+ @Override
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ SqoopOptions opts = new SqoopOptions(conf);
+ opts.setUsername(mySQLTestUtils.getUserName());
+ mySQLTestUtils.addPasswordIfIsSet(opts);
+ return opts;
+ }
+
+ @Override
+ protected void dropTableIfExists(String table) throws SQLException {
+ Connection conn = getManager().getConnection();
+ PreparedStatement statement = conn.prepareStatement(
+ "DROP TABLE IF EXISTS " + table,
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ statement.executeUpdate();
+ conn.commit();
+ } finally {
+ statement.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/mysql/MySQLLobAvroImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/mysql/MySQLLobAvroImportTest.java b/src/test/org/apache/sqoop/manager/mysql/MySQLLobAvroImportTest.java
new file mode 100644
index 0000000..a6121c9
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/mysql/MySQLLobAvroImportTest.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.mysql;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.testutil.LobAvroImportTestCase;
+
+/**
+ * Tests BLOB/CLOB import for Avro with MySQL Db.
+ */
+public class MySQLLobAvroImportTest extends LobAvroImportTestCase {
+
+ public static final Log LOG = LogFactory.getLog(
+ MySQLLobAvroImportTest.class.getName());
+ private MySQLTestUtils mySQLTestUtils = new MySQLTestUtils();
+
+ @Override
+ protected Log getLogger() {
+ return LOG;
+ }
+
+ @Override
+ protected String getDbFriendlyName() {
+ return "MySQL";
+ }
+
+ @Override
+ protected String getConnectString() {
+ return mySQLTestUtils.getMySqlConnectString();
+ }
+
+ @Override
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ SqoopOptions opts = new SqoopOptions(conf);
+ opts.setUsername(mySQLTestUtils.getUserName());
+ mySQLTestUtils.addPasswordIfIsSet(opts);
+ return opts;
+ }
+
+ @Override
+ protected void dropTableIfExists(String table) throws SQLException {
+ Connection conn = getManager().getConnection();
+ PreparedStatement statement = conn.prepareStatement(
+ "DROP TABLE IF EXISTS " + table,
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ statement.executeUpdate();
+ conn.commit();
+ } finally {
+ statement.close();
+ }
+ }
+
+ @Override
+ protected String getBlobType() {
+ return "MEDIUMBLOB";
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/mysql/MySQLTestUtils.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/mysql/MySQLTestUtils.java b/src/test/org/apache/sqoop/manager/mysql/MySQLTestUtils.java
new file mode 100644
index 0000000..25dbe9d
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/mysql/MySQLTestUtils.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.mysql;
+
+import org.apache.sqoop.SqoopOptions;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+
+/**
+ * Utilities for mysql-based tests.
+ */
+public final class MySQLTestUtils {
+
+ public static final Log LOG = LogFactory.getLog(
+ MySQLTestUtils.class.getName());
+
+ private String hostUrl;
+
+ private String userName;
+ private String userPass;
+
+ private String mysqlDbNAme;
+ private String mySqlConnectString;
+
+ public MySQLTestUtils() {
+ hostUrl = System.getProperty(
+ "sqoop.test.mysql.connectstring.host_url",
+ "jdbc:mysql://localhost/");
+ userName = System.getProperty("sqoop.test.mysql.username", getCurrentUser());
+ userPass = System.getProperty("sqoop.test.mysql.password");
+
+ mysqlDbNAme = System.getProperty("sqoop.test.mysql.databasename", "sqooptestdb");
+ mySqlConnectString = getHostUrl() + getMysqlDbNAme();
+ }
+
+ public String getHostUrl() {
+ return hostUrl;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public String getUserPass() {
+ return userPass;
+ }
+
+ public String getMysqlDbNAme() {
+ return mysqlDbNAme;
+ }
+
+
+ public String getMySqlConnectString() {
+ return mySqlConnectString;
+ }
+
+ public String[] addUserNameAndPasswordToArgs(String[] extraArgs) {
+ int extraLength = isSet(getUserPass()) ? 4 : 2;
+ String[] moreArgs = new String[extraArgs.length + extraLength];
+ int i = 0;
+ for (i = 0; i < extraArgs.length; i++) {
+ moreArgs[i] = extraArgs[i];
+ }
+
+ // Add username argument for mysql.
+ moreArgs[i++] = "--username";
+ moreArgs[i++] = getUserName();
+ if (isSet(userPass)) {
+ moreArgs[i++] = "--password";
+ moreArgs[i++] = getUserPass();
+ }
+ return moreArgs;
+ }
+
+ public static String getCurrentUser() {
+ // First, check the $USER environment variable.
+ String envUser = System.getenv("USER");
+ if (null != envUser) {
+ return envUser;
+ }
+ // Try `whoami`
+ String[] whoamiArgs = new String[1];
+ whoamiArgs[0] = "whoami";
+ Process p = null;
+ BufferedReader r = null;
+ try {
+ p = Runtime.getRuntime().exec(whoamiArgs);
+ InputStream is = p.getInputStream();
+ r = new BufferedReader(new InputStreamReader(is));
+ return r.readLine();
+ } catch (IOException ioe) {
+ LOG.error("IOException reading from `whoami`: " + ioe.toString());
+ return null;
+ } finally {
+ // close our stream.
+ if (null != r) {
+ try {
+ r.close();
+ } catch (IOException ioe) {
+ LOG.warn("IOException closing input stream from `whoami`: "
+ + ioe.toString());
+ }
+ }
+ // wait for whoami to exit.
+ while (p != null) {
+ try {
+ int ret = p.waitFor();
+ if (0 != ret) {
+ LOG.error("whoami exited with error status " + ret);
+ // suppress original return value from this method.
+ return null;
+ }
+ } catch (InterruptedException ie) {
+ continue; // loop around.
+ }
+ }
+
+ }
+ }
+
+ public void addPasswordIfIsSet(ArrayList<String> args) {
+ if (isSet(userPass)) {
+ args.add("--password");
+ args.add(getUserPass());
+ }
+ }
+
+ private boolean isSet(String userPass) {
+ return !StringUtils.isBlank(userPass);
+ }
+
+ public void addPasswordIfIsSet(SqoopOptions opts) {
+ if (isSet(userPass)) {
+ opts.setPassword(getUserPass());
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/mysql/MySqlCallExportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/mysql/MySqlCallExportTest.java b/src/test/org/apache/sqoop/manager/mysql/MySqlCallExportTest.java
index 90dff97..22a6676 100644
--- a/src/test/org/apache/sqoop/manager/mysql/MySqlCallExportTest.java
+++ b/src/test/org/apache/sqoop/manager/mysql/MySqlCallExportTest.java
@@ -34,10 +34,9 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.manager.MySQLTestUtils;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ExportJobTestCase;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.testutil.CommonArgs;
+import org.apache.sqoop.testutil.ExportJobTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/mysql/MySqlColumnEscapeImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/mysql/MySqlColumnEscapeImportTest.java b/src/test/org/apache/sqoop/manager/mysql/MySqlColumnEscapeImportTest.java
index 7ecc929..734499e 100644
--- a/src/test/org/apache/sqoop/manager/mysql/MySqlColumnEscapeImportTest.java
+++ b/src/test/org/apache/sqoop/manager/mysql/MySqlColumnEscapeImportTest.java
@@ -18,10 +18,9 @@
package org.apache.sqoop.manager.mysql;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.manager.MySQLTestUtils;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.testutil.CommonArgs;
+import org.apache.sqoop.testutil.ImportJobTestCase;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
import org.apache.commons.logging.Log;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/netezza/DirectNetezzaExportManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/netezza/DirectNetezzaExportManualTest.java b/src/test/org/apache/sqoop/manager/netezza/DirectNetezzaExportManualTest.java
index b48b379..0a6997f 100644
--- a/src/test/org/apache/sqoop/manager/netezza/DirectNetezzaExportManualTest.java
+++ b/src/test/org/apache/sqoop/manager/netezza/DirectNetezzaExportManualTest.java
@@ -27,7 +27,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.sqoop.manager.DirectNetezzaManager;
import org.junit.Test;
-import com.cloudera.sqoop.SqoopOptions;
+import org.apache.sqoop.SqoopOptions;
import static org.junit.Assert.fail;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/netezza/DirectNetezzaHCatExportManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/netezza/DirectNetezzaHCatExportManualTest.java b/src/test/org/apache/sqoop/manager/netezza/DirectNetezzaHCatExportManualTest.java
index 03cef89..9365ba0 100644
--- a/src/test/org/apache/sqoop/manager/netezza/DirectNetezzaHCatExportManualTest.java
+++ b/src/test/org/apache/sqoop/manager/netezza/DirectNetezzaHCatExportManualTest.java
@@ -33,7 +33,7 @@ import org.apache.sqoop.hcat.HCatalogTestUtils.KeyType;
import org.apache.sqoop.manager.NetezzaManager;
import org.junit.Before;
-import com.cloudera.sqoop.SqoopOptions;
+import org.apache.sqoop.SqoopOptions;
import org.junit.Test;
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/netezza/DirectNetezzaHCatImportManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/netezza/DirectNetezzaHCatImportManualTest.java b/src/test/org/apache/sqoop/manager/netezza/DirectNetezzaHCatImportManualTest.java
index ed4ae19..c05b733 100644
--- a/src/test/org/apache/sqoop/manager/netezza/DirectNetezzaHCatImportManualTest.java
+++ b/src/test/org/apache/sqoop/manager/netezza/DirectNetezzaHCatImportManualTest.java
@@ -33,7 +33,7 @@ import org.apache.sqoop.hcat.HCatalogTestUtils.KeyType;
import org.apache.sqoop.manager.NetezzaManager;
import org.junit.Before;
-import com.cloudera.sqoop.SqoopOptions;
+import org.apache.sqoop.SqoopOptions;
import org.junit.Test;
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/netezza/NetezzaExportManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/netezza/NetezzaExportManualTest.java b/src/test/org/apache/sqoop/manager/netezza/NetezzaExportManualTest.java
index 79946c5..95abe7a 100644
--- a/src/test/org/apache/sqoop/manager/netezza/NetezzaExportManualTest.java
+++ b/src/test/org/apache/sqoop/manager/netezza/NetezzaExportManualTest.java
@@ -33,10 +33,10 @@ import org.apache.sqoop.manager.DirectNetezzaManager;
import org.apache.sqoop.manager.NetezzaManager;
import org.junit.Before;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.TestExport;
-import com.cloudera.sqoop.testutil.BaseSqoopTestCase;
-import com.cloudera.sqoop.testutil.CommonArgs;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.TestExport;
+import org.apache.sqoop.testutil.BaseSqoopTestCase;
+import org.apache.sqoop.testutil.CommonArgs;
import static org.junit.Assert.fail;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/netezza/NetezzaImportManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/netezza/NetezzaImportManualTest.java b/src/test/org/apache/sqoop/manager/netezza/NetezzaImportManualTest.java
index 1adbbdb..4002c64 100644
--- a/src/test/org/apache/sqoop/manager/netezza/NetezzaImportManualTest.java
+++ b/src/test/org/apache/sqoop/manager/netezza/NetezzaImportManualTest.java
@@ -39,11 +39,11 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.manager.ConnManager;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
-import com.cloudera.sqoop.util.FileListing;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.manager.ConnManager;
+import org.apache.sqoop.testutil.CommonArgs;
+import org.apache.sqoop.testutil.ImportJobTestCase;
+import org.apache.sqoop.util.FileListing;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/netezza/NetezzaTestUtils.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/netezza/NetezzaTestUtils.java b/src/test/org/apache/sqoop/manager/netezza/NetezzaTestUtils.java
index 1abdb59..00deb8f 100644
--- a/src/test/org/apache/sqoop/manager/netezza/NetezzaTestUtils.java
+++ b/src/test/org/apache/sqoop/manager/netezza/NetezzaTestUtils.java
@@ -26,7 +26,7 @@ import java.sql.SQLException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.cloudera.sqoop.TestExport.ColumnGenerator;
+import org.apache.sqoop.TestExport.ColumnGenerator;
/**
* Utilities for Netezza tests.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/oracle/OraOopDataDrivenDBInputFormatConnectionCloseTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OraOopDataDrivenDBInputFormatConnectionCloseTest.java b/src/test/org/apache/sqoop/manager/oracle/OraOopDataDrivenDBInputFormatConnectionCloseTest.java
index 8e31c3f..bb33c35 100644
--- a/src/test/org/apache/sqoop/manager/oracle/OraOopDataDrivenDBInputFormatConnectionCloseTest.java
+++ b/src/test/org/apache/sqoop/manager/oracle/OraOopDataDrivenDBInputFormatConnectionCloseTest.java
@@ -18,7 +18,7 @@
package org.apache.sqoop.manager.oracle;
-import com.cloudera.sqoop.mapreduce.db.DBConfiguration;
+import org.apache.sqoop.mapreduce.db.DBConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobContext;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/oracle/OraOopOracleQueriesTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OraOopOracleQueriesTest.java b/src/test/org/apache/sqoop/manager/oracle/OraOopOracleQueriesTest.java
index abd3329..dca2772 100644
--- a/src/test/org/apache/sqoop/manager/oracle/OraOopOracleQueriesTest.java
+++ b/src/test/org/apache/sqoop/manager/oracle/OraOopOracleQueriesTest.java
@@ -25,7 +25,7 @@ import java.sql.PreparedStatement;
import org.junit.Test;
-import com.cloudera.sqoop.manager.OracleUtils;
+import org.apache.sqoop.manager.oracle.util.OracleUtils;
/**
* Test Oracle queries against Oracle database.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java b/src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java
index 56dcac6..1bae71c 100644
--- a/src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java
+++ b/src/test/org/apache/sqoop/manager/oracle/OraOopTestCase.java
@@ -26,7 +26,6 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -43,8 +42,8 @@ import org.apache.log4j.WriterAppender;
import org.apache.sqoop.manager.oracle.util.HadoopFiles;
import org.apache.sqoop.manager.oracle.util.OracleData;
-import com.cloudera.sqoop.Sqoop;
-import com.cloudera.sqoop.manager.OracleUtils;
+import org.apache.sqoop.Sqoop;
+import org.apache.sqoop.manager.oracle.util.OracleUtils;
import static org.junit.Assert.assertEquals;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/oracle/OracleCallExportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OracleCallExportTest.java b/src/test/org/apache/sqoop/manager/oracle/OracleCallExportTest.java
index a473f67..6d6602a 100644
--- a/src/test/org/apache/sqoop/manager/oracle/OracleCallExportTest.java
+++ b/src/test/org/apache/sqoop/manager/oracle/OracleCallExportTest.java
@@ -34,10 +34,10 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.manager.OracleUtils;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ExportJobTestCase;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.manager.oracle.util.OracleUtils;
+import org.apache.sqoop.testutil.CommonArgs;
+import org.apache.sqoop.testutil.ExportJobTestCase;
import org.junit.Test;
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/oracle/OracleColumnEscapeImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OracleColumnEscapeImportTest.java b/src/test/org/apache/sqoop/manager/oracle/OracleColumnEscapeImportTest.java
index 1e3b799..d4146dc 100644
--- a/src/test/org/apache/sqoop/manager/oracle/OracleColumnEscapeImportTest.java
+++ b/src/test/org/apache/sqoop/manager/oracle/OracleColumnEscapeImportTest.java
@@ -18,10 +18,10 @@
package org.apache.sqoop.manager.oracle;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.manager.OracleUtils;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.manager.oracle.util.OracleUtils;
+import org.apache.sqoop.testutil.CommonArgs;
+import org.apache.sqoop.testutil.ImportJobTestCase;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
import org.apache.commons.logging.Log;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/oracle/OracleCompatTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OracleCompatTest.java b/src/test/org/apache/sqoop/manager/oracle/OracleCompatTest.java
new file mode 100644
index 0000000..553096a
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/oracle/OracleCompatTest.java
@@ -0,0 +1,269 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.oracle;
+
+import java.io.UnsupportedEncodingException;
+import java.sql.SQLException;
+import java.util.Formatter;
+
+import org.apache.sqoop.manager.oracle.util.OracleUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.testutil.ManagerCompatTestCase;
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
+
+/**
+ * Test the basic Oracle connection manager with the various column types.
+ */
+public class OracleCompatTest extends ManagerCompatTestCase {
+
+ public static final Log LOG = LogFactory.getLog(
+ OracleCompatTest.class.getName());
+
+ @Override
+ protected Log getLogger() {
+ return LOG;
+ }
+
+ @Override
+ protected String getDbFriendlyName() {
+ return "Oracle";
+ }
+
+ @Override
+ protected String getConnectString() {
+ return OracleUtils.CONNECT_STRING;
+ }
+
+ @Override
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ SqoopOptions opts = new SqoopOptions(conf);
+ OracleUtils.setOracleAuth(opts);
+ return opts;
+
+ }
+
+ @Override
+ protected void dropTableIfExists(String table) throws SQLException {
+ OracleUtils.dropTable(table, getManager());
+ }
+
+ @Override
+ public void tearDown() {
+ super.tearDown();
+
+ // If we actually ran the test, we'll need to 'cool off' afterwards.
+ if (!skipped) {
+ // Oracle XE will block connections if you create new ones too quickly.
+ // See http://forums.oracle.com/forums/thread.jspa?messageID=1145120
+ LOG.info("Sleeping to wait for Oracle connection cache clear...");
+ try {
+ Thread.sleep(250);
+ } catch (InterruptedException ie) {
+ // This delay may run a bit short.. no problem.
+ }
+ }
+ }
+
+ @Override
+ protected String getDoubleType() {
+ return "DOUBLE PRECISION";
+ }
+
+ @Override
+ protected String getVarBinaryType() {
+ return "RAW(12)";
+ }
+
+ // Oracle does not provide a BOOLEAN type.
+ @Override
+ protected boolean supportsBoolean() {
+ return false;
+ }
+
+ // Oracle does not provide a BIGINT type.
+ @Override
+ protected boolean supportsBigInt() {
+ return false;
+ }
+
+ // Oracle does not provide a TINYINT type.
+ @Override
+ protected boolean supportsTinyInt() {
+ return false;
+ }
+
+ // Oracle does not provide a LONGVARCHAR type.
+ @Override
+ protected boolean supportsLongVarChar() {
+ return false;
+ }
+
+ // Oracle does not provide a TIME type. We test DATE and TIMESTAMP
+ @Override
+ protected boolean supportsTime() {
+ return false;
+ }
+
+ @Override
+ protected String getDateInsertStr(String dateStr) {
+ return "TO_DATE(" + dateStr + ", 'YYYY-MM-DD')";
+ }
+
+ @Override
+ protected String getTimestampInsertStr(String tsStr) {
+ return "TO_TIMESTAMP(" + tsStr + ", 'YYYY-MM-DD HH24:MI:SS.FF')";
+ }
+
+ @Override
+ protected String getDateSeqOutput(String asInserted) {
+ // DATE is actually a TIMESTAMP in Oracle; add a time component.
+ return asInserted + " 00:00:00.0";
+ }
+
+ @Override
+ protected String getFixedCharSeqOut(int fieldWidth, String asInserted) {
+ return padString(fieldWidth, asInserted);
+ }
+
+ @Override
+ protected String getRealSeqOutput(String realAsInserted) {
+ return realAsInserted;
+ }
+
+ @Override
+ protected String getFloatSeqOutput(String floatAsInserted) {
+ return floatAsInserted;
+ }
+
+ @Override
+ protected String getDoubleSeqOutput(String doubleAsInserted) {
+ return doubleAsInserted;
+ }
+
+ @Override
+ protected String getVarBinarySeqOutput(String asInserted) {
+ return toLowerHexString(asInserted);
+ }
+
+ @Override
+ protected String getBlobInsertStr(String blobData) {
+ // Oracle wants blob data encoded as hex (e.g. '01fca3b5').
+
+ StringBuilder sb = new StringBuilder();
+ sb.append("'");
+
+ Formatter fmt = new Formatter(sb);
+ try {
+ for (byte b : blobData.getBytes("UTF-8")) {
+ fmt.format("%02X", b);
+ }
+ } catch (UnsupportedEncodingException uee) {
+ // Should not happen; Java always supports UTF-8.
+ fail("Could not get utf-8 bytes for blob string");
+ return null;
+ }
+ sb.append("'");
+ return sb.toString();
+ }
+
+ protected String getBinaryFloatInsertStr(float f) {
+ return "TO_BINARY_FLOAT('" + f + "')";
+ }
+
+ protected String getBinaryDoubleInsertStr(double d) {
+ return "TO_BINARY_DOUBLE('" + d + "')";
+ }
+
+ // Disable this test since Oracle isn't ANSI compliant.
+ @Override
+ @Test
+ public void testEmptyStringCol() {
+ this.skipped = true;
+ LOG.info(
+ "Oracle treats empty strings as null (non-ANSI compliant). Skipping.");
+ }
+
+ @Override
+ @Test
+ public void testTimestamp1() {
+ verifyType(getTimestampType(),
+ getTimestampInsertStr("'2009-04-24 18:24:00'"),
+ "2009-04-24 18:24:00.0");
+ }
+
+ @Override
+ @Test
+ public void testTimestamp2() {
+ verifyType(getTimestampType(),
+ getTimestampInsertStr("'2009-04-24 18:24:00.0002'"),
+ "2009-04-24 18:24:00.0002");
+ }
+
+ @Override
+ @Test
+ public void testDate1() {
+ verifyType("DATE", getDateInsertStr("'2009-01-12'"),
+ getDateSeqOutput("2009-01-12"));
+ }
+
+ @Override
+ @Test
+ public void testDate2() {
+ verifyType("DATE", getDateInsertStr("'2009-04-24'"),
+ getDateSeqOutput("2009-04-24"));
+ }
+
+ @Test
+ public void testRawVal() {
+ verifyType("RAW(8)", "'12ABCD'", getVarBinarySeqOutput("12ABCD"), true);
+ }
+
+ @Test
+ public void testBinaryFloat() {
+ verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(25f), "25.0");
+ verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(+6.34f), "6.34");
+
+ // Max and min are from Oracle DB SQL reference for 10g release 2
+ float max = 3.40282E+38F;
+ verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(max), "3.40282E38");
+ float min = 1.17549E-38F;
+ verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(min), "1.17549E-38");
+ }
+
+ @Test
+ public void testBinaryDouble() {
+ verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(0.5d), "0.5");
+ verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(-1d), "-1.0");
+
+ // Max and min are from Oracle DB SQL reference for 10g release 2
+ double max = 1.79769313486231E+308;
+ verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(max),
+ "1.79769313486231E308");
+ double min = 2.22507485850720E-308;
+ verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(min),
+ "2.2250748585072E-308");
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/oracle/OracleConnectionFactoryTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OracleConnectionFactoryTest.java b/src/test/org/apache/sqoop/manager/oracle/OracleConnectionFactoryTest.java
index 9e6931b..34e182f 100644
--- a/src/test/org/apache/sqoop/manager/oracle/OracleConnectionFactoryTest.java
+++ b/src/test/org/apache/sqoop/manager/oracle/OracleConnectionFactoryTest.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.conf.Configuration;
import org.junit.Assert;
import org.junit.Test;
-import com.cloudera.sqoop.manager.OracleUtils;
+import org.apache.sqoop.manager.oracle.util.OracleUtils;
/**
* Test OracleConnectionFactory class including initialization statements.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/oracle/OracleExportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OracleExportTest.java b/src/test/org/apache/sqoop/manager/oracle/OracleExportTest.java
new file mode 100644
index 0000000..a880af3
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/oracle/OracleExportTest.java
@@ -0,0 +1,315 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.oracle;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.SQLException;
+
+import org.apache.sqoop.manager.oracle.util.OracleUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.sqoop.manager.OracleManager;
+import org.junit.After;
+import org.junit.Before;
+
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.TestExport;
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
+
+/**
+ * Test the OracleManager implementation's exportJob() functionality.
+ * This tests the OracleExportOutputFormat (which subclasses
+ * ExportOutputFormat with Oracle-specific SQL statements).
+ */
+public class OracleExportTest extends TestExport {
+
+ public static final Log LOG = LogFactory.getLog(
+ OracleExportTest.class.getName());
+
+ static final String TABLE_PREFIX = "EXPORT_ORACLE_";
+
+ // instance variables populated during setUp, used during tests.
+ private OracleManager manager;
+ private Connection conn;
+
+ @Override
+ protected Connection getConnection() {
+ return conn;
+ }
+
+ // Oracle allows multi-row inserts (with its own syntax).
+ @Override
+ protected int getMaxRowsPerStatement() {
+ return 1000;
+ }
+
+ @Override
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ @Override
+ protected String getConnectString() {
+ return OracleUtils.CONNECT_STRING;
+ }
+
+ @Override
+ protected String getTablePrefix() {
+ return TABLE_PREFIX;
+ }
+
+ @Override
+ protected String getDropTableStatement(String tableName) {
+ return OracleUtils.getDropTableStatement(tableName);
+ }
+
+ @Before
+ public void setUp() {
+ super.setUp();
+
+ SqoopOptions options = new SqoopOptions(OracleUtils.CONNECT_STRING,
+ getTableName());
+ OracleUtils.setOracleAuth(options);
+ this.manager = new OracleManager(options);
+ try {
+ this.conn = manager.getConnection();
+ this.conn.setAutoCommit(false);
+ } catch (SQLException sqlE) {
+ LOG.error(StringUtils.stringifyException(sqlE));
+ fail("Failed with sql exception in setup: " + sqlE);
+ }
+ }
+
+ @After
+ public void tearDown() {
+ super.tearDown();
+
+ if (null != this.conn) {
+ try {
+ this.conn.close();
+ } catch (SQLException sqlE) {
+ LOG.error("Got SQLException closing conn: " + sqlE.toString());
+ }
+ }
+
+ if (null != manager) {
+ try {
+ manager.close();
+ manager = null;
+ } catch (SQLException sqlE) {
+ LOG.error("Got SQLException: " + sqlE.toString());
+ fail("Got SQLException: " + sqlE.toString());
+ }
+ }
+ }
+
+ @Override
+ protected String [] getCodeGenArgv(String... extraArgs) {
+ String [] moreArgs = new String[extraArgs.length + 4];
+ int i = 0;
+ for (i = 0; i < extraArgs.length; i++) {
+ moreArgs[i] = extraArgs[i];
+ }
+
+ // Add username and password args.
+ moreArgs[i++] = "--username";
+ moreArgs[i++] = OracleUtils.ORACLE_USER_NAME;
+ moreArgs[i++] = "--password";
+ moreArgs[i++] = OracleUtils.ORACLE_USER_PASS;
+
+ return super.getCodeGenArgv(moreArgs);
+ }
+
+ @Override
+ protected String [] getArgv(boolean includeHadoopFlags,
+ int rowsPerStatement, int statementsPerTx, String... additionalArgv) {
+
+ String [] subArgv = newStrArray(additionalArgv,
+ "--username", OracleUtils.ORACLE_USER_NAME,
+ "--password", OracleUtils.ORACLE_USER_PASS);
+ return super.getArgv(includeHadoopFlags, rowsPerStatement,
+ statementsPerTx, subArgv);
+ }
+
+ @Override
+ protected ColumnGenerator getDateColumnGenerator() {
+ // Return a TIMESTAMP generator that has increasing date values.
+ // We currently do not support export of DATE columns since
+ // Oracle informs us that they are actually TIMESTAMP; this messes
+ // up Sqoop's parsing of values as we have no way of knowing they're
+ // really supposed to be dates based on the Oracle Jdbc Metadata.
+ return new ColumnGenerator() {
+ public String getExportText(int rowNum) {
+ int day = rowNum + 1;
+ return "2009-10-" + pad(day) + " 00:00:00.0";
+ }
+ public String getVerifyText(int rowNum) {
+ int day = rowNum + 1;
+ return "2009-10-" + day + " 0:0:0. 0";
+ }
+ public String getType() {
+ return "TIMESTAMP"; // TODO: Support DATE more intelligently.
+ }
+ };
+ }
+
+ @Override
+ protected ColumnGenerator getTimeColumnGenerator() {
+ // Return a TIMESTAMP generator that has increasing time values.
+ // We currently do not support the export of DATE columns with
+ // only a time component set (because Oracle reports these column
+ // types to Sqoop as TIMESTAMP, and we parse the user's text
+ // incorrectly based on this result).
+ return new ColumnGenerator() {
+ public String getExportText(int rowNum) {
+ return "1970-01-01 10:01:" + pad(rowNum) + ".0";
+ }
+ public String getVerifyText(int rowNum) {
+ return "1970-1-1 10:1:" + rowNum + ". 0";
+ }
+ public String getType() {
+ return "TIMESTAMP";
+ }
+ };
+ }
+
+ protected ColumnGenerator getNewDateColGenerator() {
+ // Return a TIMESTAMP generator that has increasing date values.
+ // Use the "new" Oracle string output format
+ return new ColumnGenerator() {
+ public String getExportText(int rowNum) {
+ int day = rowNum + 1;
+ return "2009-10-" + pad(day) + " 00:00:00.0";
+ }
+ public String getVerifyText(int rowNum) {
+ int day = rowNum + 1;
+ return "2009-10-" + pad(day) + " 00:00:00";
+ }
+ public String getType() {
+ return "TIMESTAMP"; // TODO: Support DATE more intelligently.
+ }
+ };
+ }
+
+ protected ColumnGenerator getNewTimeColGenerator() {
+ // Return a TIMESTAMP generator that has increasing time values.
+ // We currently do not support the export of DATE columns with
+ // only a time component set (because Oracle reports these column
+ // types to Sqoop as TIMESTAMP, and we parse the user's text
+ // incorrectly based on this result).
+ return new ColumnGenerator() {
+ public String getExportText(int rowNum) {
+ return "1970-01-01 10:01:" + pad(rowNum) + ".0";
+ }
+ public String getVerifyText(int rowNum) {
+ return "1970-01-01 10:01:" + pad(rowNum);
+ }
+ public String getType() {
+ return "TIMESTAMP";
+ }
+ };
+ }
+
+ @Override
+ protected String getBigIntType() {
+ // Oracle stores everything in NUMERIC columns.
+ return "NUMERIC(12,0)";
+ }
+
+ // Dates and times seem to be formatted differently in different
+ // versions of Oracle's JDBC jar. We run this test twice with
+ // different versions of the column generators to check whether
+ // either one succeeds.
+ @Override
+ public void testDatesAndTimes() throws IOException, SQLException {
+ final int TOTAL_RECORDS = 10;
+
+ ColumnGenerator genDate = getDateColumnGenerator();
+ ColumnGenerator genTime = getTimeColumnGenerator();
+
+ try {
+ createTextFile(0, TOTAL_RECORDS, false, genDate, genTime);
+ createTable(genDate, genTime);
+ runExport(getArgv(true, 10, 10));
+ verifyExport(TOTAL_RECORDS);
+ assertColMinAndMax(forIdx(0), genDate);
+ assertColMinAndMax(forIdx(1), genTime);
+ } catch (AssertionError afe) {
+ genDate = getNewDateColGenerator();
+ genTime = getNewTimeColGenerator();
+
+ createTextFile(0, TOTAL_RECORDS, false, genDate, genTime);
+ createTable(genDate, genTime);
+ runExport(getArgv(true, 10, 10));
+ verifyExport(TOTAL_RECORDS);
+ assertColMinAndMax(forIdx(0), genDate);
+ assertColMinAndMax(forIdx(1), genTime);
+ }
+ }
+
+ /** Make sure mixed update/insert export work correctly. */
+ @Test
+ public void testUpsertTextExport() throws IOException, SQLException {
+ final int TOTAL_RECORDS = 10;
+ createTextFile(0, TOTAL_RECORDS, false);
+ createTable();
+ // first time will be insert.
+ runExport(getArgv(true, 10, 10, newStrArray(null,
+ "--update-key", "ID", "--update-mode", "allowinsert", "--oracle-escaping-disabled", "false")));
+ // second time will be update.
+ runExport(getArgv(true, 10, 10, newStrArray(null,
+ "--update-key", "ID", "--update-mode", "allowinsert", "--oracle-escaping-disabled", "false")));
+ verifyExport(TOTAL_RECORDS);
+ }
+
+ /** Make sure mixed update/insert export work correctly. */
+ @Test
+ public void testUpsertTextExportWithEscapingDisabled() throws IOException, SQLException {
+ final int TOTAL_RECORDS = 10;
+ createTextFile(0, TOTAL_RECORDS, false);
+ createTable();
+ // first time will be insert.
+ runExport(getArgv(true, 10, 10, newStrArray(null,
+ "--update-key", "ID", "--update-mode", "allowinsert", "--oracle-escaping-disabled", "true")));
+ // second time will be update.
+ runExport(getArgv(true, 10, 10, newStrArray(null,
+ "--update-key", "ID", "--update-mode", "allowinsert", "--oracle-escaping-disabled", "true")));
+ verifyExport(TOTAL_RECORDS);
+ }
+
+ @Test
+ public void testExportToTableWithNameEndingWithDollarSign() throws IOException, SQLException {
+ testExportToTableWithName("DOLLAR$");
+ }
+
+ @Test
+ public void testExportToTableWithNameContainingDollarSign() throws IOException, SQLException {
+ testExportToTableWithName("FOO$BAR");
+ }
+
+ @Test
+ public void testExportToTableWithNameContainingHashtag() throws IOException, SQLException {
+ testExportToTableWithName("FOO#BAR");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/oracle/OracleFreeFormQueryTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OracleFreeFormQueryTest.java b/src/test/org/apache/sqoop/manager/oracle/OracleFreeFormQueryTest.java
new file mode 100644
index 0000000..bb3e7c4
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/oracle/OracleFreeFormQueryTest.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.oracle;
+
+import java.sql.SQLException;
+
+import org.apache.sqoop.manager.oracle.util.OracleUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.TestFreeFormQueryImport;
+
+/**
+ * Test free form query import with the Oracle db.
+ */
+public class OracleFreeFormQueryTest extends TestFreeFormQueryImport {
+
+ public static final Log LOG = LogFactory.getLog(
+ OracleFreeFormQueryTest.class.getName());
+
+ @Override
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ @Override
+ protected String getConnectString() {
+ return OracleUtils.CONNECT_STRING;
+ }
+
+ @Override
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ SqoopOptions opts = new SqoopOptions(conf);
+ OracleUtils.setOracleAuth(opts);
+ return opts;
+ }
+
+ @Override
+ protected void dropTableIfExists(String table) throws SQLException {
+ OracleUtils.dropTable(table, getManager());
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/oracle/OracleIncrementalImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OracleIncrementalImportTest.java b/src/test/org/apache/sqoop/manager/oracle/OracleIncrementalImportTest.java
index 2a908b3..4dde3d6 100644
--- a/src/test/org/apache/sqoop/manager/oracle/OracleIncrementalImportTest.java
+++ b/src/test/org/apache/sqoop/manager/oracle/OracleIncrementalImportTest.java
@@ -18,10 +18,10 @@
package org.apache.sqoop.manager.oracle;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.manager.OracleUtils;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.manager.oracle.util.OracleUtils;
+import org.apache.sqoop.testutil.CommonArgs;
+import org.apache.sqoop.testutil.ImportJobTestCase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/oracle/OracleLobAvroImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OracleLobAvroImportTest.java b/src/test/org/apache/sqoop/manager/oracle/OracleLobAvroImportTest.java
new file mode 100644
index 0000000..525ccf4
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/oracle/OracleLobAvroImportTest.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.oracle;
+
+import java.io.UnsupportedEncodingException;
+import java.sql.SQLException;
+import java.util.Formatter;
+
+import org.apache.sqoop.manager.oracle.util.OracleUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.testutil.LobAvroImportTestCase;
+
+import static org.junit.Assert.fail;
+
+/**
+ * Tests BLOB/CLOB import for Avro with Oracle Db.
+ */
+public class OracleLobAvroImportTest extends LobAvroImportTestCase {
+
+ public static final Log LOG = LogFactory.getLog(
+ OracleCompatTest.class.getName());
+
+ @Override
+ protected Log getLogger() {
+ return LOG;
+ }
+
+ @Override
+ protected String getDbFriendlyName() {
+ return "Oracle";
+ }
+
+ @Override
+ protected String getConnectString() {
+ return OracleUtils.CONNECT_STRING;
+ }
+
+ @Override
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ SqoopOptions opts = new SqoopOptions(conf);
+ OracleUtils.setOracleAuth(opts);
+ return opts;
+ }
+
+ @Override
+ protected void dropTableIfExists(String table) throws SQLException {
+ OracleUtils.dropTable(table, getManager());
+ }
+
+ @Override
+ protected String getBlobInsertStr(String blobData) {
+ // Oracle wants blob data encoded as hex (e.g. '01fca3b5').
+
+ StringBuilder sb = new StringBuilder();
+ sb.append("'");
+
+ Formatter fmt = new Formatter(sb);
+ try {
+ for (byte b : blobData.getBytes("UTF-8")) {
+ fmt.format("%02X", b);
+ }
+ } catch (UnsupportedEncodingException uee) {
+ // Should not happen; Java always supports UTF-8.
+ fail("Could not get utf-8 bytes for blob string");
+ return null;
+ }
+ sb.append("'");
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/oracle/OracleManagerTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OracleManagerTest.java b/src/test/org/apache/sqoop/manager/oracle/OracleManagerTest.java
new file mode 100644
index 0000000..9251f02
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/oracle/OracleManagerTest.java
@@ -0,0 +1,560 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.oracle;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.FileInputStream;
+import java.io.File;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Date;
+import java.util.Calendar;
+import java.util.TimeZone;
+import java.util.ArrayList;
+import java.text.ParseException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+
+import org.apache.sqoop.manager.oracle.util.OracleUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.sqoop.manager.ConnManager;
+import org.apache.sqoop.manager.OracleManager;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.testutil.CommonArgs;
+import org.apache.sqoop.testutil.ImportJobTestCase;
+import org.apache.sqoop.util.FileListing;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test the OracleManager implementation.
+ *
+ * This uses JDBC to import data from an Oracle database into HDFS.
+ *
+ * Since this requires an Oracle installation on your local machine to use,
+ * this class is named in such a way that Hadoop's default QA process does
+ * not run it. You need to run this manually with -Dtestcase=OracleManagerTest.
+ *
+ * You need to put Oracle's JDBC driver library (ojdbc6_g.jar) in a location
+ * where Hadoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons).
+ *
+ * To set up your test environment:
+ * <ol>
+ * <li>Install Oracle 10.2.0 or later</li>
+ * <li>Create a database user named SQOOPTEST with password '12345' having
+ * CONNECT and RESOURCE privileges.</li>
+ * <li>Create a database user named SQOOPTEST2 with password 'abcdef' having
+ * CONNECT and RESOURCE privileges</li>
+ * </ol>
+ *
+ * One way to do this is to connect to the database instance via SQL*Plus client
+ * as the SYSTEM user and execute the following commands:
+ * <ul>
+ * <li>CREATE USER SQOOPTEST identified by 12345;</li>
+ * <li>GRANT CONNECT, RESOURCE to SQOOPTEST;</li>
+ * <li>CREATE USER SQOOPTEST2 identified by ABCDEF;</li>
+ * <li>GRANT CONNECT, RESOURCE to SQOOPTEST2;</li>
+ * </ul>
+ *
+ * Oracle XE does a poor job of cleaning up connections in a timely fashion.
+ * Too many connections too quickly will be rejected, because XE will gc the
+ * closed connections in a lazy fashion. Oracle tests have a delay built in
+ * to work with this gc, but it is possible that you will see this error:
+ *
+ * ORA-12516, TNS:listener could not find available handler with matching
+ * protocol stack
+ *
+ * If so, log in to your database as SYSTEM and execute the following:
+ * ALTER SYSTEM SET processes=200 scope=spfile;
+ * ... then restart your database.
+ */
+public class OracleManagerTest extends ImportJobTestCase {
+
+ public static final Log LOG = LogFactory.getLog(
+ OracleManagerTest.class.getName());
+
+ static final String TABLE_NAME = "EMPLOYEES";
+ static final String SECONDARY_TABLE_NAME = "CUSTOMER";
+ static final String QUALIFIED_SECONDARY_TABLE_NAME =
+ OracleUtils.ORACLE_SECONDARY_USER_NAME + "." + SECONDARY_TABLE_NAME;
+
+ /*
+ * Array containing SQL statements necessary to create and populate
+ * the main test table.
+ */
+ private static final String[] MAIN_TABLE_SQL_STMTS = new String[] {
+ "CREATE TABLE " + TABLE_NAME + " ("
+ + "id INT NOT NULL, "
+ + "name VARCHAR2(24) NOT NULL, "
+ + "start_date DATE, "
+ + "salary FLOAT, "
+ + "dept VARCHAR2(32), "
+ + "timestamp_tz TIMESTAMP WITH TIME ZONE, "
+ + "timestamp_ltz TIMESTAMP WITH LOCAL TIME ZONE, "
+ + "PRIMARY KEY (id))",
+ "INSERT INTO " + TABLE_NAME + " VALUES("
+ + "1,'Aaron',to_date('2009-05-14','yyyy-mm-dd'),"
+ + "1000000.00,'engineering','29-DEC-09 12.00.00.000000000 PM',"
+ + "'29-DEC-09 12.00.00.000000000 PM')",
+ "INSERT INTO " + TABLE_NAME + " VALUES("
+ + "2,'Bob',to_date('2009-04-20','yyyy-mm-dd'),"
+ + "400.00,'sales','30-DEC-09 12.00.00.000000000 PM',"
+ + "'30-DEC-09 12.00.00.000000000 PM')",
+ "INSERT INTO " + TABLE_NAME + " VALUES("
+ + "3,'Fred',to_date('2009-01-23','yyyy-mm-dd'),15.00,"
+ + "'marketing','31-DEC-09 12.00.00.000000000 PM',"
+ + "'31-DEC-09 12.00.00.000000000 PM')",
+ };
+
+ /*
+ * Array containing SQL statements necessary to create, populate and
+ * provision the secondary test table.
+ */
+ private static final String[] SECONDARY_TABLE_SQL_STMTS = new String[] {
+ "CREATE TABLE " + SECONDARY_TABLE_NAME + " ("
+ + "id INT NOT NULL, "
+ + "name VARCHAR2(24) NOT NULL, "
+ + "PRIMARY KEY (id))",
+ "INSERT INTO " + SECONDARY_TABLE_NAME + " VALUES("
+ + "1,'MercuryCorp')",
+ "INSERT INTO " + SECONDARY_TABLE_NAME + " VALUES("
+ + "2,'VenusCorp')",
+ "INSERT INTO " + SECONDARY_TABLE_NAME + " VALUES("
+ + "3,'EarthCorp')",
+ "INSERT INTO " + SECONDARY_TABLE_NAME + " VALUES("
+ + "4,'MarsCorp')",
+ "INSERT INTO " + SECONDARY_TABLE_NAME + " VALUES("
+ + "5,'JupiterCorp')",
+ "INSERT INTO " + SECONDARY_TABLE_NAME + " VALUES("
+ + "6,'SaturnCorp')",
+ "GRANT SELECT, INSERT ON " + SECONDARY_TABLE_NAME + " TO "
+ + OracleUtils.ORACLE_USER_NAME,
+ };
+
+ // instance variables populated during setUp, used during tests
+ private OracleManager manager;
+
+ @Override
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ private void executeUpdates(OracleManager mgr, String[] sqlStmts) {
+ Connection connection = null;
+ Statement st = null;
+
+ try {
+ connection = mgr.getConnection();
+ connection.setAutoCommit(false);
+ st = connection.createStatement();
+
+ for (String sql : sqlStmts) {
+ st.executeUpdate(sql);
+ }
+ connection.commit();
+ } catch (SQLException sqlE) {
+ LOG.error("Encountered SQL Exception: " + sqlE);
+ sqlE.printStackTrace();
+ fail("SQLException when running test setUp(): " + sqlE);
+ } finally {
+ try {
+ if (null != st) {
+ st.close();
+ }
+
+ if (null != connection) {
+ connection.close();
+ }
+ } catch (SQLException sqlE) {
+ LOG.warn("Got SQLException when closing connection: " + sqlE);
+ }
+ }
+ }
+
+ private void provisionSecondaryTable() {
+ SqoopOptions options = new SqoopOptions(OracleUtils.CONNECT_STRING,
+ SECONDARY_TABLE_NAME);
+ OracleUtils.setOracleSecondaryUserAuth(options);
+
+ OracleManager mgr = new OracleManager(options);
+
+ // Drop the existing table if there is one
+ try {
+ OracleUtils.dropTable(SECONDARY_TABLE_NAME, mgr);
+ } catch (SQLException sqlE) {
+ fail("Could not drop table " + SECONDARY_TABLE_NAME + ": " + sqlE);
+ }
+
+ executeUpdates(mgr, SECONDARY_TABLE_SQL_STMTS);
+
+ try {
+ mgr.close();
+ } catch (SQLException sqlE) {
+ fail("Failed to close secondary manager instance : " + sqlE);
+ }
+ }
+
+
+ @Before
+ public void setUp() {
+ super.setUp();
+
+ provisionSecondaryTable();
+
+ SqoopOptions options = new SqoopOptions(OracleUtils.CONNECT_STRING,
+ TABLE_NAME);
+ OracleUtils.setOracleAuth(options);
+
+ manager = new OracleManager(options);
+ options.getConf().set("oracle.sessionTimeZone", "US/Pacific");
+
+ // Drop the existing table, if there is one.
+ try {
+ OracleUtils.dropTable(TABLE_NAME, manager);
+ } catch (SQLException sqlE) {
+ fail("Could not drop table " + TABLE_NAME + ": " + sqlE);
+ }
+
+ executeUpdates(manager, MAIN_TABLE_SQL_STMTS);
+ }
+
+ @After
+ public void tearDown() {
+ super.tearDown();
+ try {
+ manager.close();
+ } catch (SQLException sqlE) {
+ LOG.error("Got SQLException: " + sqlE.toString());
+ fail("Got SQLException: " + sqlE.toString());
+ }
+ }
+
+ private String[] getArgv() {
+ return getArgv(TABLE_NAME);
+ }
+
+ private String [] getArgv(String tableName) {
+ ArrayList<String> args = new ArrayList<String>();
+
+ CommonArgs.addHadoopFlags(args);
+
+ args.add("-D");
+ args.add("oracle.sessionTimeZone=US/Pacific");
+
+ args.add("--table");
+ args.add(tableName);
+ args.add("--warehouse-dir");
+ args.add(getWarehouseDir());
+ args.add("--connect");
+ args.add(OracleUtils.CONNECT_STRING);
+ args.add("--username");
+ args.add(OracleUtils.ORACLE_USER_NAME);
+ args.add("--password");
+ args.add(OracleUtils.ORACLE_USER_PASS);
+ args.add("--num-mappers");
+ args.add("1");
+
+ return args.toArray(new String[0]);
+ }
+
+ private void runSecondaryTableTest(String [] expectedResults)
+ throws IOException {
+ Path warehousePath = new Path(this.getWarehouseDir());
+ Path tablePath = new Path(warehousePath, QUALIFIED_SECONDARY_TABLE_NAME);
+ Path filePath = new Path(tablePath, "part-m-00000");
+
+ File tableFile = new File(tablePath.toString());
+ if (tableFile.exists() && tableFile.isDirectory()) {
+ // remove the directory before running the import
+ FileListing.recursiveDeleteDir(tableFile);
+ }
+
+ String [] argv = getArgv(QUALIFIED_SECONDARY_TABLE_NAME);
+
+ try {
+ runImport(argv);
+ } catch (IOException ioe) {
+ LOG.error("Got IOException during import: " + ioe.toString());
+ ioe.printStackTrace();
+ fail(ioe.toString());
+ }
+
+ File f = new File(filePath.toString());
+ assertTrue("Could not find imported data file", f.exists());
+ BufferedReader r = null;
+ try {
+ // Read through the file and make sure it's all there.
+ r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
+ for (String expectedLine : expectedResults) {
+ compareRecords(expectedLine, r.readLine());
+ }
+ } catch (IOException ioe) {
+ LOG.error("Got IOException verifying results: " + ioe.toString());
+ ioe.printStackTrace();
+ fail(ioe.toString());
+ } finally {
+ IOUtils.closeStream(r);
+ }
+ }
+
+ private void runOracleTest(String [] expectedResults) throws IOException {
+
+ Path warehousePath = new Path(this.getWarehouseDir());
+ Path tablePath = new Path(warehousePath, TABLE_NAME);
+ Path filePath = new Path(tablePath, "part-m-00000");
+
+ File tableFile = new File(tablePath.toString());
+ if (tableFile.exists() && tableFile.isDirectory()) {
+ // remove the directory before running the import.
+ FileListing.recursiveDeleteDir(tableFile);
+ }
+
+ String [] argv = getArgv();
+ try {
+ runImport(argv);
+ } catch (IOException ioe) {
+ LOG.error("Got IOException during import: " + ioe.toString());
+ ioe.printStackTrace();
+ fail(ioe.toString());
+ }
+
+ File f = new File(filePath.toString());
+ assertTrue("Could not find imported data file", f.exists());
+ BufferedReader r = null;
+ try {
+ // Read through the file and make sure it's all there.
+ r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
+ for (String expectedLine : expectedResults) {
+ compareRecords(expectedLine, r.readLine());
+ }
+ } catch (IOException ioe) {
+ LOG.error("Got IOException verifying results: " + ioe.toString());
+ ioe.printStackTrace();
+ fail(ioe.toString());
+ } finally {
+ IOUtils.closeStream(r);
+ }
+ }
+
+ @Test
+ public void testOracleImport() throws IOException {
+ // no quoting of strings allowed. NOTE: Oracle JDBC 11.1 drivers
+ // auto-cast SQL DATE to java.sql.Timestamp. Even if you define your
+ // columns as DATE in Oracle, they may still contain time information, so
+ // the JDBC drivers lie to us and will never tell us we have a strict DATE
+ // type. Thus we include HH:MM:SS.mmmmm below.
+ // See http://www.oracle.com
+ // /technology/tech/java/sqlj_jdbc/htdocs/jdbc_faq.html#08_01
+ String [] expectedResults = {
+ "1,Aaron,2009-05-14 00:00:00.0,1000000,engineering,"
+ + "2009-12-29 12:00:00.0,2009-12-29 12:00:00.0",
+ "2,Bob,2009-04-20 00:00:00.0,400,sales,"
+ + "2009-12-30 12:00:00.0,2009-12-30 12:00:00.0",
+ "3,Fred,2009-01-23 00:00:00.0,15,marketing,"
+ + "2009-12-31 12:00:00.0,2009-12-31 12:00:00.0",
+ };
+
+ runOracleTest(expectedResults);
+ }
+
+ @Test
+ public void testSecondaryTableImport() throws IOException {
+
+ String [] expectedResults = {
+ "1,MercuryCorp",
+ "2,VenusCorp",
+ "3,EarthCorp",
+ "4,MarsCorp",
+ "5,JupiterCorp",
+ "6,SaturnCorp",
+ };
+ runSecondaryTableTest(expectedResults);
+ }
+
+ /**
+ * Compare two lines. Normalize the dates we receive based on the expected
+ * time zone.
+ * @param expectedLine expected line
+ * @param receivedLine received line
+ * @throws IOException exception during lines comparison
+ */
+ private void compareRecords(String expectedLine, String receivedLine)
+ throws IOException {
+ // handle null case
+ if (expectedLine == null || receivedLine == null) {
+ return;
+ }
+
+ // check if lines are equal
+ if (expectedLine.equals(receivedLine)) {
+ return;
+ }
+
+ // check if size is the same
+ String [] expectedValues = expectedLine.split(",");
+ String [] receivedValues = receivedLine.split(",");
+ if (expectedValues.length != 7 || receivedValues.length != 7) {
+ LOG.error("Number of expected fields did not match "
+ + "number of received fields");
+ throw new IOException("Number of expected fields did not match "
+ + "number of received fields");
+ }
+
+ // check first 5 values
+ boolean mismatch = false;
+ for (int i = 0; !mismatch && i < 5; i++) {
+ mismatch = !expectedValues[i].equals(receivedValues[i]);
+ }
+ if (mismatch) {
+ throw new IOException("Expected:<" + expectedLine + "> but was:<"
+ + receivedLine + ">");
+ }
+
+ Date expectedDate = null;
+ Date receivedDate = null;
+ DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S");
+ int offset = TimeZone.getDefault().getOffset(System.currentTimeMillis())
+ / 3600000;
+ for (int i = 5; i < 7; i++) {
+ // parse expected timestamp.
+ try {
+ expectedDate = df.parse(expectedValues[i]);
+ } catch (ParseException ex) {
+ LOG.error("Could not parse expected timestamp: " + expectedValues[i]);
+ throw new IOException("Could not parse expected timestamp: "
+ + expectedValues[i]);
+ }
+
+ // parse received timestamp
+ try {
+ receivedDate = df.parse(receivedValues[i]);
+ } catch (ParseException ex) {
+ LOG.error("Could not parse received timestamp: " + receivedValues[i]);
+ throw new IOException("Could not parse received timestamp: "
+ + receivedValues[i]);
+ }
+
+ // compare two timestamps considering timezone offset
+ Calendar expectedCal = Calendar.getInstance();
+ expectedCal.setTime(expectedDate);
+ expectedCal.add(Calendar.HOUR, offset);
+
+ Calendar receivedCal = Calendar.getInstance();
+ receivedCal.setTime(receivedDate);
+
+ if (!expectedCal.equals(receivedCal)) {
+ throw new IOException("Expected:<" + expectedLine + "> but was:<"
+ + receivedLine + ">, while timezone offset is: " + offset);
+ }
+ }
+ }
+
+ @Test
+ public void testPurgeClosedConnections() throws Exception {
+ // Ensure that after an Oracle ConnManager releases any connections
+ // back into the cache (or closes them as redundant), it does not
+ // attempt to re-use the closed connection.
+
+ SqoopOptions options = new SqoopOptions(OracleUtils.CONNECT_STRING,
+ TABLE_NAME);
+ OracleUtils.setOracleAuth(options);
+
+ // Create a connection manager, use it, and then recycle its connection
+ // into the cache.
+ ConnManager m1 = new OracleManager(options);
+ Connection c1 = m1.getConnection();
+ PreparedStatement s = c1.prepareStatement("SELECT 1 FROM dual",
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ ResultSet rs = null;
+ try {
+ rs = s.executeQuery();
+ rs.close();
+ } finally {
+ s.close();
+ }
+
+ ConnManager m2 = new OracleManager(options);
+ Connection c2 = m2.getConnection(); // get a new connection.
+
+ m1.close(); // c1 should now be cached.
+
+ // Use the second connection to run a statement.
+ s = c2.prepareStatement("SELECT 2 FROM dual",
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ rs = s.executeQuery();
+ rs.close();
+ } finally {
+ s.close();
+ }
+
+ m2.close(); // c2 should be discarded (c1 is already cached).
+
+ // Try to get another connection from m2. This should result in
+ // a completely different connection getting served back to us.
+ Connection c2a = m2.getConnection();
+
+ assertFalse(c1.isClosed());
+ assertTrue(c2.isClosed());
+ assertFalse(c2a.isClosed());
+
+ s = c2a.prepareStatement("SELECT 3 FROM dual",
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ rs = s.executeQuery();
+ rs.close();
+ } finally {
+ s.close();
+ }
+
+ m2.close(); // Close the manager's active connection again.
+ }
+
+ @Test
+ public void testSessionUserName() throws Exception {
+ SqoopOptions options = new SqoopOptions(OracleUtils.CONNECT_STRING,
+ TABLE_NAME);
+ OracleUtils.setOracleAuth(options);
+
+ // Create a connection manager and get a connection
+ OracleManager m1 = new OracleManager(options);
+ Connection c1 = m1.getConnection();
+ // Make sure that the session username is the same as the Oracle
+ // sqoop user name
+ String sessionUserName = m1.getSessionUser(c1);
+ Assert.assertEquals(OracleUtils.ORACLE_USER_NAME, sessionUserName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/oracle/OracleSpecialCharacterTableImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OracleSpecialCharacterTableImportTest.java b/src/test/org/apache/sqoop/manager/oracle/OracleSpecialCharacterTableImportTest.java
index 907be49..453ad82 100644
--- a/src/test/org/apache/sqoop/manager/oracle/OracleSpecialCharacterTableImportTest.java
+++ b/src/test/org/apache/sqoop/manager/oracle/OracleSpecialCharacterTableImportTest.java
@@ -18,10 +18,10 @@
package org.apache.sqoop.manager.oracle;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.manager.OracleUtils;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.manager.oracle.util.OracleUtils;
+import org.apache.sqoop.testutil.CommonArgs;
+import org.apache.sqoop.testutil.ImportJobTestCase;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
import org.apache.commons.logging.Log;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/oracle/OracleSplitterTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OracleSplitterTest.java b/src/test/org/apache/sqoop/manager/oracle/OracleSplitterTest.java
index 6878608..33b7d74 100644
--- a/src/test/org/apache/sqoop/manager/oracle/OracleSplitterTest.java
+++ b/src/test/org/apache/sqoop/manager/oracle/OracleSplitterTest.java
@@ -18,28 +18,18 @@
package org.apache.sqoop.manager.oracle;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.manager.OracleUtils;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.manager.oracle.util.OracleUtils;
+import org.apache.sqoop.testutil.CommonArgs;
+import org.apache.sqoop.testutil.ImportJobTestCase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
import org.junit.Test;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Writer;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;