You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by an...@apache.org on 2018/01/18 14:04:04 UTC
[18/32] sqoop git commit: SQOOP-3273: Removing com.cloudera.sqoop
packages
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/OracleCompatTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/OracleCompatTest.java b/src/test/com/cloudera/sqoop/manager/OracleCompatTest.java
deleted file mode 100644
index 0d615e3..0000000
--- a/src/test/com/cloudera/sqoop/manager/OracleCompatTest.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.manager;
-
-import java.io.UnsupportedEncodingException;
-import java.sql.SQLException;
-import java.util.Formatter;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.testutil.ManagerCompatTestCase;
-import org.junit.Test;
-
-import static org.junit.Assert.fail;
-
-/**
- * Test the basic Oracle connection manager with the various column types.
- */
-public class OracleCompatTest extends ManagerCompatTestCase {
-
- public static final Log LOG = LogFactory.getLog(
- OracleCompatTest.class.getName());
-
- @Override
- protected Log getLogger() {
- return LOG;
- }
-
- @Override
- protected String getDbFriendlyName() {
- return "Oracle";
- }
-
- @Override
- protected String getConnectString() {
- return OracleUtils.CONNECT_STRING;
- }
-
- @Override
- protected SqoopOptions getSqoopOptions(Configuration conf) {
- SqoopOptions opts = new SqoopOptions(conf);
- OracleUtils.setOracleAuth(opts);
- return opts;
-
- }
-
- @Override
- protected void dropTableIfExists(String table) throws SQLException {
- OracleUtils.dropTable(table, getManager());
- }
-
- @Override
- public void tearDown() {
- super.tearDown();
-
- // If we actually ran the test, we'll need to 'cool off' afterwards.
- if (!skipped) {
- // Oracle XE will block connections if you create new ones too quickly.
- // See http://forums.oracle.com/forums/thread.jspa?messageID=1145120
- LOG.info("Sleeping to wait for Oracle connection cache clear...");
- try {
- Thread.sleep(250);
- } catch (InterruptedException ie) {
- // This delay may run a bit short.. no problem.
- }
- }
- }
-
- @Override
- protected String getDoubleType() {
- return "DOUBLE PRECISION";
- }
-
- @Override
- protected String getVarBinaryType() {
- return "RAW(12)";
- }
-
- // Oracle does not provide a BOOLEAN type.
- @Override
- protected boolean supportsBoolean() {
- return false;
- }
-
- // Oracle does not provide a BIGINT type.
- @Override
- protected boolean supportsBigInt() {
- return false;
- }
-
- // Oracle does not provide a TINYINT type.
- @Override
- protected boolean supportsTinyInt() {
- return false;
- }
-
- // Oracle does not provide a LONGVARCHAR type.
- @Override
- protected boolean supportsLongVarChar() {
- return false;
- }
-
- // Oracle does not provide a TIME type. We test DATE and TIMESTAMP
- @Override
- protected boolean supportsTime() {
- return false;
- }
-
- @Override
- protected String getDateInsertStr(String dateStr) {
- return "TO_DATE(" + dateStr + ", 'YYYY-MM-DD')";
- }
-
- @Override
- protected String getTimestampInsertStr(String tsStr) {
- return "TO_TIMESTAMP(" + tsStr + ", 'YYYY-MM-DD HH24:MI:SS.FF')";
- }
-
- @Override
- protected String getDateSeqOutput(String asInserted) {
- // DATE is actually a TIMESTAMP in Oracle; add a time component.
- return asInserted + " 00:00:00.0";
- }
-
- @Override
- protected String getFixedCharSeqOut(int fieldWidth, String asInserted) {
- return padString(fieldWidth, asInserted);
- }
-
- @Override
- protected String getRealSeqOutput(String realAsInserted) {
- return realAsInserted;
- }
-
- @Override
- protected String getFloatSeqOutput(String floatAsInserted) {
- return floatAsInserted;
- }
-
- @Override
- protected String getDoubleSeqOutput(String doubleAsInserted) {
- return doubleAsInserted;
- }
-
- @Override
- protected String getVarBinarySeqOutput(String asInserted) {
- return toLowerHexString(asInserted);
- }
-
- @Override
- protected String getBlobInsertStr(String blobData) {
- // Oracle wants blob data encoded as hex (e.g. '01fca3b5').
-
- StringBuilder sb = new StringBuilder();
- sb.append("'");
-
- Formatter fmt = new Formatter(sb);
- try {
- for (byte b : blobData.getBytes("UTF-8")) {
- fmt.format("%02X", b);
- }
- } catch (UnsupportedEncodingException uee) {
- // Should not happen; Java always supports UTF-8.
- fail("Could not get utf-8 bytes for blob string");
- return null;
- }
- sb.append("'");
- return sb.toString();
- }
-
- protected String getBinaryFloatInsertStr(float f) {
- return "TO_BINARY_FLOAT('" + f + "')";
- }
-
- protected String getBinaryDoubleInsertStr(double d) {
- return "TO_BINARY_DOUBLE('" + d + "')";
- }
-
- // Disable this test since Oracle isn't ANSI compliant.
- @Override
- @Test
- public void testEmptyStringCol() {
- this.skipped = true;
- LOG.info(
- "Oracle treats empty strings as null (non-ANSI compliant). Skipping.");
- }
-
- @Override
- @Test
- public void testTimestamp1() {
- verifyType(getTimestampType(),
- getTimestampInsertStr("'2009-04-24 18:24:00'"),
- "2009-04-24 18:24:00.0");
- }
-
- @Override
- @Test
- public void testTimestamp2() {
- verifyType(getTimestampType(),
- getTimestampInsertStr("'2009-04-24 18:24:00.0002'"),
- "2009-04-24 18:24:00.0002");
- }
-
- @Override
- @Test
- public void testDate1() {
- verifyType("DATE", getDateInsertStr("'2009-01-12'"),
- getDateSeqOutput("2009-01-12"));
- }
-
- @Override
- @Test
- public void testDate2() {
- verifyType("DATE", getDateInsertStr("'2009-04-24'"),
- getDateSeqOutput("2009-04-24"));
- }
-
- @Test
- public void testRawVal() {
- verifyType("RAW(8)", "'12ABCD'", getVarBinarySeqOutput("12ABCD"), true);
- }
-
- @Test
- public void testBinaryFloat() {
- verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(25f), "25.0");
- verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(+6.34f), "6.34");
-
- // Max and min are from Oracle DB SQL reference for 10g release 2
- float max = 3.40282E+38F;
- verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(max), "3.40282E38");
- float min = 1.17549E-38F;
- verifyType("BINARY_FLOAT", getBinaryFloatInsertStr(min), "1.17549E-38");
- }
-
- @Test
- public void testBinaryDouble() {
- verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(0.5d), "0.5");
- verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(-1d), "-1.0");
-
- // Max and min are from Oracle DB SQL reference for 10g release 2
- double max = 1.79769313486231E+308;
- verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(max),
- "1.79769313486231E308");
- double min = 2.22507485850720E-308;
- verifyType("BINARY_DOUBLE", getBinaryDoubleInsertStr(min),
- "2.2250748585072E-308");
- }
-}
-
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/OracleExportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/OracleExportTest.java b/src/test/com/cloudera/sqoop/manager/OracleExportTest.java
deleted file mode 100644
index fe2e265..0000000
--- a/src/test/com/cloudera/sqoop/manager/OracleExportTest.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.manager;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.SQLException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.StringUtils;
-import org.junit.After;
-import org.junit.Before;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.TestExport;
-import org.junit.Test;
-
-import static org.junit.Assert.fail;
-
-/**
- * Test the OracleManager implementation's exportJob() functionality.
- * This tests the OracleExportOutputFormat (which subclasses
- * ExportOutputFormat with Oracle-specific SQL statements).
- */
-public class OracleExportTest extends TestExport {
-
- public static final Log LOG = LogFactory.getLog(
- OracleExportTest.class.getName());
-
- static final String TABLE_PREFIX = "EXPORT_ORACLE_";
-
- // instance variables populated during setUp, used during tests.
- private OracleManager manager;
- private Connection conn;
-
- @Override
- protected Connection getConnection() {
- return conn;
- }
-
- // Oracle allows multi-row inserts (with its own syntax).
- @Override
- protected int getMaxRowsPerStatement() {
- return 1000;
- }
-
- @Override
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- @Override
- protected String getConnectString() {
- return OracleUtils.CONNECT_STRING;
- }
-
- @Override
- protected String getTablePrefix() {
- return TABLE_PREFIX;
- }
-
- @Override
- protected String getDropTableStatement(String tableName) {
- return OracleUtils.getDropTableStatement(tableName);
- }
-
- @Before
- public void setUp() {
- super.setUp();
-
- SqoopOptions options = new SqoopOptions(OracleUtils.CONNECT_STRING,
- getTableName());
- OracleUtils.setOracleAuth(options);
- this.manager = new OracleManager(options);
- try {
- this.conn = manager.getConnection();
- this.conn.setAutoCommit(false);
- } catch (SQLException sqlE) {
- LOG.error(StringUtils.stringifyException(sqlE));
- fail("Failed with sql exception in setup: " + sqlE);
- }
- }
-
- @After
- public void tearDown() {
- super.tearDown();
-
- if (null != this.conn) {
- try {
- this.conn.close();
- } catch (SQLException sqlE) {
- LOG.error("Got SQLException closing conn: " + sqlE.toString());
- }
- }
-
- if (null != manager) {
- try {
- manager.close();
- manager = null;
- } catch (SQLException sqlE) {
- LOG.error("Got SQLException: " + sqlE.toString());
- fail("Got SQLException: " + sqlE.toString());
- }
- }
- }
-
- @Override
- protected String [] getCodeGenArgv(String... extraArgs) {
- String [] moreArgs = new String[extraArgs.length + 4];
- int i = 0;
- for (i = 0; i < extraArgs.length; i++) {
- moreArgs[i] = extraArgs[i];
- }
-
- // Add username and password args.
- moreArgs[i++] = "--username";
- moreArgs[i++] = OracleUtils.ORACLE_USER_NAME;
- moreArgs[i++] = "--password";
- moreArgs[i++] = OracleUtils.ORACLE_USER_PASS;
-
- return super.getCodeGenArgv(moreArgs);
- }
-
- @Override
- protected String [] getArgv(boolean includeHadoopFlags,
- int rowsPerStatement, int statementsPerTx, String... additionalArgv) {
-
- String [] subArgv = newStrArray(additionalArgv,
- "--username", OracleUtils.ORACLE_USER_NAME,
- "--password", OracleUtils.ORACLE_USER_PASS);
- return super.getArgv(includeHadoopFlags, rowsPerStatement,
- statementsPerTx, subArgv);
- }
-
- @Override
- protected ColumnGenerator getDateColumnGenerator() {
- // Return a TIMESTAMP generator that has increasing date values.
- // We currently do not support export of DATE columns since
- // Oracle informs us that they are actually TIMESTAMP; this messes
- // up Sqoop's parsing of values as we have no way of knowing they're
- // really supposed to be dates based on the Oracle Jdbc Metadata.
- return new ColumnGenerator() {
- public String getExportText(int rowNum) {
- int day = rowNum + 1;
- return "2009-10-" + pad(day) + " 00:00:00.0";
- }
- public String getVerifyText(int rowNum) {
- int day = rowNum + 1;
- return "2009-10-" + day + " 0:0:0. 0";
- }
- public String getType() {
- return "TIMESTAMP"; // TODO: Support DATE more intelligently.
- }
- };
- }
-
- @Override
- protected ColumnGenerator getTimeColumnGenerator() {
- // Return a TIMESTAMP generator that has increasing time values.
- // We currently do not support the export of DATE columns with
- // only a time component set (because Oracle reports these column
- // types to Sqoop as TIMESTAMP, and we parse the user's text
- // incorrectly based on this result).
- return new ColumnGenerator() {
- public String getExportText(int rowNum) {
- return "1970-01-01 10:01:" + pad(rowNum) + ".0";
- }
- public String getVerifyText(int rowNum) {
- return "1970-1-1 10:1:" + rowNum + ". 0";
- }
- public String getType() {
- return "TIMESTAMP";
- }
- };
- }
-
- protected ColumnGenerator getNewDateColGenerator() {
- // Return a TIMESTAMP generator that has increasing date values.
- // Use the "new" Oracle string output format
- return new ColumnGenerator() {
- public String getExportText(int rowNum) {
- int day = rowNum + 1;
- return "2009-10-" + pad(day) + " 00:00:00.0";
- }
- public String getVerifyText(int rowNum) {
- int day = rowNum + 1;
- return "2009-10-" + pad(day) + " 00:00:00";
- }
- public String getType() {
- return "TIMESTAMP"; // TODO: Support DATE more intelligently.
- }
- };
- }
-
- protected ColumnGenerator getNewTimeColGenerator() {
- // Return a TIMESTAMP generator that has increasing time values.
- // We currently do not support the export of DATE columns with
- // only a time component set (because Oracle reports these column
- // types to Sqoop as TIMESTAMP, and we parse the user's text
- // incorrectly based on this result).
- return new ColumnGenerator() {
- public String getExportText(int rowNum) {
- return "1970-01-01 10:01:" + pad(rowNum) + ".0";
- }
- public String getVerifyText(int rowNum) {
- return "1970-01-01 10:01:" + pad(rowNum);
- }
- public String getType() {
- return "TIMESTAMP";
- }
- };
- }
-
- @Override
- protected String getBigIntType() {
- // Oracle stores everything in NUMERIC columns.
- return "NUMERIC(12,0)";
- }
-
- // Dates and times seem to be formatted differently in different
- // versions of Oracle's JDBC jar. We run this test twice with
- // different versions of the column generators to check whether
- // either one succeeds.
- @Override
- public void testDatesAndTimes() throws IOException, SQLException {
- final int TOTAL_RECORDS = 10;
-
- ColumnGenerator genDate = getDateColumnGenerator();
- ColumnGenerator genTime = getTimeColumnGenerator();
-
- try {
- createTextFile(0, TOTAL_RECORDS, false, genDate, genTime);
- createTable(genDate, genTime);
- runExport(getArgv(true, 10, 10));
- verifyExport(TOTAL_RECORDS);
- assertColMinAndMax(forIdx(0), genDate);
- assertColMinAndMax(forIdx(1), genTime);
- } catch (AssertionError afe) {
- genDate = getNewDateColGenerator();
- genTime = getNewTimeColGenerator();
-
- createTextFile(0, TOTAL_RECORDS, false, genDate, genTime);
- createTable(genDate, genTime);
- runExport(getArgv(true, 10, 10));
- verifyExport(TOTAL_RECORDS);
- assertColMinAndMax(forIdx(0), genDate);
- assertColMinAndMax(forIdx(1), genTime);
- }
- }
-
- /** Make sure mixed update/insert export work correctly. */
- @Test
- public void testUpsertTextExport() throws IOException, SQLException {
- final int TOTAL_RECORDS = 10;
- createTextFile(0, TOTAL_RECORDS, false);
- createTable();
- // first time will be insert.
- runExport(getArgv(true, 10, 10, newStrArray(null,
- "--update-key", "ID", "--update-mode", "allowinsert", "--oracle-escaping-disabled", "false")));
- // second time will be update.
- runExport(getArgv(true, 10, 10, newStrArray(null,
- "--update-key", "ID", "--update-mode", "allowinsert", "--oracle-escaping-disabled", "false")));
- verifyExport(TOTAL_RECORDS);
- }
-
- /** Make sure mixed update/insert export work correctly. */
- @Test
- public void testUpsertTextExportWithEscapingDisabled() throws IOException, SQLException {
- final int TOTAL_RECORDS = 10;
- createTextFile(0, TOTAL_RECORDS, false);
- createTable();
- // first time will be insert.
- runExport(getArgv(true, 10, 10, newStrArray(null,
- "--update-key", "ID", "--update-mode", "allowinsert", "--oracle-escaping-disabled", "true")));
- // second time will be update.
- runExport(getArgv(true, 10, 10, newStrArray(null,
- "--update-key", "ID", "--update-mode", "allowinsert", "--oracle-escaping-disabled", "true")));
- verifyExport(TOTAL_RECORDS);
- }
-
- @Test
- public void testExportToTableWithNameEndingWithDollarSign() throws IOException, SQLException {
- testExportToTableWithName("DOLLAR$");
- }
-
- @Test
- public void testExportToTableWithNameContainingDollarSign() throws IOException, SQLException {
- testExportToTableWithName("FOO$BAR");
- }
-
- @Test
- public void testExportToTableWithNameContainingHashtag() throws IOException, SQLException {
- testExportToTableWithName("FOO#BAR");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/OracleFreeFormQueryTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/OracleFreeFormQueryTest.java b/src/test/com/cloudera/sqoop/manager/OracleFreeFormQueryTest.java
deleted file mode 100644
index e4f9423..0000000
--- a/src/test/com/cloudera/sqoop/manager/OracleFreeFormQueryTest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.manager;
-
-import java.sql.SQLException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.TestFreeFormQueryImport;
-
-/**
- * Test free form query import with the Oracle db.
- */
-public class OracleFreeFormQueryTest extends TestFreeFormQueryImport {
-
- public static final Log LOG = LogFactory.getLog(
- OracleFreeFormQueryTest.class.getName());
-
- @Override
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- @Override
- protected String getConnectString() {
- return OracleUtils.CONNECT_STRING;
- }
-
- @Override
- protected SqoopOptions getSqoopOptions(Configuration conf) {
- SqoopOptions opts = new SqoopOptions(conf);
- OracleUtils.setOracleAuth(opts);
- return opts;
- }
-
- @Override
- protected void dropTableIfExists(String table) throws SQLException {
- OracleUtils.dropTable(table, getManager());
- }
-}
-
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/OracleLobAvroImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/OracleLobAvroImportTest.java b/src/test/com/cloudera/sqoop/manager/OracleLobAvroImportTest.java
deleted file mode 100644
index a23f088..0000000
--- a/src/test/com/cloudera/sqoop/manager/OracleLobAvroImportTest.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.manager;
-
-import java.io.UnsupportedEncodingException;
-import java.sql.SQLException;
-import java.util.Formatter;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.testutil.LobAvroImportTestCase;
-
-import static org.junit.Assert.fail;
-
-/**
- * Tests BLOB/CLOB import for Avro with Oracle Db.
- */
-public class OracleLobAvroImportTest extends LobAvroImportTestCase {
-
- public static final Log LOG = LogFactory.getLog(
- OracleCompatTest.class.getName());
-
- @Override
- protected Log getLogger() {
- return LOG;
- }
-
- @Override
- protected String getDbFriendlyName() {
- return "Oracle";
- }
-
- @Override
- protected String getConnectString() {
- return OracleUtils.CONNECT_STRING;
- }
-
- @Override
- protected SqoopOptions getSqoopOptions(Configuration conf) {
- SqoopOptions opts = new SqoopOptions(conf);
- OracleUtils.setOracleAuth(opts);
- return opts;
- }
-
- @Override
- protected void dropTableIfExists(String table) throws SQLException {
- OracleUtils.dropTable(table, getManager());
- }
-
- @Override
- protected String getBlobInsertStr(String blobData) {
- // Oracle wants blob data encoded as hex (e.g. '01fca3b5').
-
- StringBuilder sb = new StringBuilder();
- sb.append("'");
-
- Formatter fmt = new Formatter(sb);
- try {
- for (byte b : blobData.getBytes("UTF-8")) {
- fmt.format("%02X", b);
- }
- } catch (UnsupportedEncodingException uee) {
- // Should not happen; Java always supports UTF-8.
- fail("Could not get utf-8 bytes for blob string");
- return null;
- }
- sb.append("'");
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/OracleManagerTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/OracleManagerTest.java b/src/test/com/cloudera/sqoop/manager/OracleManagerTest.java
deleted file mode 100644
index 817141b..0000000
--- a/src/test/com/cloudera/sqoop/manager/OracleManagerTest.java
+++ /dev/null
@@ -1,556 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.manager;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.FileInputStream;
-import java.io.File;
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.Statement;
-import java.util.Date;
-import java.util.Calendar;
-import java.util.TimeZone;
-import java.util.ArrayList;
-import java.text.ParseException;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
-import com.cloudera.sqoop.util.FileListing;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Test the OracleManager implementation.
- *
- * This uses JDBC to import data from an Oracle database into HDFS.
- *
- * Since this requires an Oracle installation on your local machine to use,
- * this class is named in such a way that Hadoop's default QA process does
- * not run it. You need to run this manually with -Dtestcase=OracleManagerTest.
- *
- * You need to put Oracle's JDBC driver library (ojdbc6_g.jar) in a location
- * where Hadoop will be able to access it (since this library cannot be checked
- * into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * <ol>
- * <li>Install Oracle 10.2.0 or later</li>
- * <li>Create a database user named SQOOPTEST with password '12345' having
- * CONNECT and RESOURCE privileges.</li>
- * <li>Create a database user named SQOOPTEST2 with password 'abcdef' having
- * CONNECT and RESOURCE privileges</li>
- * </ol>
- *
- * One way to do this is to connect to the database instance via SQL*Plus client
- * as the SYSTEM user and execute the following commands:
- * <ul>
- * <li>CREATE USER SQOOPTEST identified by 12345;</li>
- * <li>GRANT CONNECT, RESOURCE to SQOOPTEST;</li>
- * <li>CREATE USER SQOOPTEST2 identified by ABCDEF;</li>
- * <li>GRANT CONNECT, RESOURCE to SQOOPTEST2;</li>
- * </ul>
- *
- * Oracle XE does a poor job of cleaning up connections in a timely fashion.
- * Too many connections too quickly will be rejected, because XE will gc the
- * closed connections in a lazy fashion. Oracle tests have a delay built in
- * to work with this gc, but it is possible that you will see this error:
- *
- * ORA-12516, TNS:listener could not find available handler with matching
- * protocol stack
- *
- * If so, log in to your database as SYSTEM and execute the following:
- * ALTER SYSTEM SET processes=200 scope=spfile;
- * ... then restart your database.
- */
-public class OracleManagerTest extends ImportJobTestCase {
-
- public static final Log LOG = LogFactory.getLog(
- OracleManagerTest.class.getName());
-
- static final String TABLE_NAME = "EMPLOYEES";
- static final String SECONDARY_TABLE_NAME = "CUSTOMER";
- static final String QUALIFIED_SECONDARY_TABLE_NAME =
- OracleUtils.ORACLE_SECONDARY_USER_NAME + "." + SECONDARY_TABLE_NAME;
-
- /*
- * Array containing SQL statements necessary to create and populate
- * the main test table.
- */
- private static final String[] MAIN_TABLE_SQL_STMTS = new String[] {
- "CREATE TABLE " + TABLE_NAME + " ("
- + "id INT NOT NULL, "
- + "name VARCHAR2(24) NOT NULL, "
- + "start_date DATE, "
- + "salary FLOAT, "
- + "dept VARCHAR2(32), "
- + "timestamp_tz TIMESTAMP WITH TIME ZONE, "
- + "timestamp_ltz TIMESTAMP WITH LOCAL TIME ZONE, "
- + "PRIMARY KEY (id))",
- "INSERT INTO " + TABLE_NAME + " VALUES("
- + "1,'Aaron',to_date('2009-05-14','yyyy-mm-dd'),"
- + "1000000.00,'engineering','29-DEC-09 12.00.00.000000000 PM',"
- + "'29-DEC-09 12.00.00.000000000 PM')",
- "INSERT INTO " + TABLE_NAME + " VALUES("
- + "2,'Bob',to_date('2009-04-20','yyyy-mm-dd'),"
- + "400.00,'sales','30-DEC-09 12.00.00.000000000 PM',"
- + "'30-DEC-09 12.00.00.000000000 PM')",
- "INSERT INTO " + TABLE_NAME + " VALUES("
- + "3,'Fred',to_date('2009-01-23','yyyy-mm-dd'),15.00,"
- + "'marketing','31-DEC-09 12.00.00.000000000 PM',"
- + "'31-DEC-09 12.00.00.000000000 PM')",
- };
-
- /*
- * Array containing SQL statements necessary to create, populate and
- * provision the secondary test table.
- */
- private static final String[] SECONDARY_TABLE_SQL_STMTS = new String[] {
- "CREATE TABLE " + SECONDARY_TABLE_NAME + " ("
- + "id INT NOT NULL, "
- + "name VARCHAR2(24) NOT NULL, "
- + "PRIMARY KEY (id))",
- "INSERT INTO " + SECONDARY_TABLE_NAME + " VALUES("
- + "1,'MercuryCorp')",
- "INSERT INTO " + SECONDARY_TABLE_NAME + " VALUES("
- + "2,'VenusCorp')",
- "INSERT INTO " + SECONDARY_TABLE_NAME + " VALUES("
- + "3,'EarthCorp')",
- "INSERT INTO " + SECONDARY_TABLE_NAME + " VALUES("
- + "4,'MarsCorp')",
- "INSERT INTO " + SECONDARY_TABLE_NAME + " VALUES("
- + "5,'JupiterCorp')",
- "INSERT INTO " + SECONDARY_TABLE_NAME + " VALUES("
- + "6,'SaturnCorp')",
- "GRANT SELECT, INSERT ON " + SECONDARY_TABLE_NAME + " TO "
- + OracleUtils.ORACLE_USER_NAME,
- };
-
- // instance variables populated during setUp, used during tests
- private OracleManager manager;
-
- @Override
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- private void executeUpdates(OracleManager mgr, String[] sqlStmts) {
- Connection connection = null;
- Statement st = null;
-
- try {
- connection = mgr.getConnection();
- connection.setAutoCommit(false);
- st = connection.createStatement();
-
- for (String sql : sqlStmts) {
- st.executeUpdate(sql);
- }
- connection.commit();
- } catch (SQLException sqlE) {
- LOG.error("Encountered SQL Exception: " + sqlE);
- sqlE.printStackTrace();
- fail("SQLException when running test setUp(): " + sqlE);
- } finally {
- try {
- if (null != st) {
- st.close();
- }
-
- if (null != connection) {
- connection.close();
- }
- } catch (SQLException sqlE) {
- LOG.warn("Got SQLException when closing connection: " + sqlE);
- }
- }
- }
-
- private void provisionSecondaryTable() {
- SqoopOptions options = new SqoopOptions(OracleUtils.CONNECT_STRING,
- SECONDARY_TABLE_NAME);
- OracleUtils.setOracleSecondaryUserAuth(options);
-
- OracleManager mgr = new OracleManager(options);
-
- // Drop the existing table if there is one
- try {
- OracleUtils.dropTable(SECONDARY_TABLE_NAME, mgr);
- } catch (SQLException sqlE) {
- fail("Could not drop table " + SECONDARY_TABLE_NAME + ": " + sqlE);
- }
-
- executeUpdates(mgr, SECONDARY_TABLE_SQL_STMTS);
-
- try {
- mgr.close();
- } catch (SQLException sqlE) {
- fail("Failed to close secondary manager instance : " + sqlE);
- }
- }
-
-
- @Before
- public void setUp() {
- super.setUp();
-
- provisionSecondaryTable();
-
- SqoopOptions options = new SqoopOptions(OracleUtils.CONNECT_STRING,
- TABLE_NAME);
- OracleUtils.setOracleAuth(options);
-
- manager = new OracleManager(options);
- options.getConf().set("oracle.sessionTimeZone", "US/Pacific");
-
- // Drop the existing table, if there is one.
- try {
- OracleUtils.dropTable(TABLE_NAME, manager);
- } catch (SQLException sqlE) {
- fail("Could not drop table " + TABLE_NAME + ": " + sqlE);
- }
-
- executeUpdates(manager, MAIN_TABLE_SQL_STMTS);
- }
-
- @After
- public void tearDown() {
- super.tearDown();
- try {
- manager.close();
- } catch (SQLException sqlE) {
- LOG.error("Got SQLException: " + sqlE.toString());
- fail("Got SQLException: " + sqlE.toString());
- }
- }
-
- private String[] getArgv() {
- return getArgv(TABLE_NAME);
- }
-
- private String [] getArgv(String tableName) {
- ArrayList<String> args = new ArrayList<String>();
-
- CommonArgs.addHadoopFlags(args);
-
- args.add("-D");
- args.add("oracle.sessionTimeZone=US/Pacific");
-
- args.add("--table");
- args.add(tableName);
- args.add("--warehouse-dir");
- args.add(getWarehouseDir());
- args.add("--connect");
- args.add(OracleUtils.CONNECT_STRING);
- args.add("--username");
- args.add(OracleUtils.ORACLE_USER_NAME);
- args.add("--password");
- args.add(OracleUtils.ORACLE_USER_PASS);
- args.add("--num-mappers");
- args.add("1");
-
- return args.toArray(new String[0]);
- }
-
- private void runSecondaryTableTest(String [] expectedResults)
- throws IOException {
- Path warehousePath = new Path(this.getWarehouseDir());
- Path tablePath = new Path(warehousePath, QUALIFIED_SECONDARY_TABLE_NAME);
- Path filePath = new Path(tablePath, "part-m-00000");
-
- File tableFile = new File(tablePath.toString());
- if (tableFile.exists() && tableFile.isDirectory()) {
- // remove the directory before running the import
- FileListing.recursiveDeleteDir(tableFile);
- }
-
- String [] argv = getArgv(QUALIFIED_SECONDARY_TABLE_NAME);
-
- try {
- runImport(argv);
- } catch (IOException ioe) {
- LOG.error("Got IOException during import: " + ioe.toString());
- ioe.printStackTrace();
- fail(ioe.toString());
- }
-
- File f = new File(filePath.toString());
- assertTrue("Could not find imported data file", f.exists());
- BufferedReader r = null;
- try {
- // Read through the file and make sure it's all there.
- r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
- for (String expectedLine : expectedResults) {
- compareRecords(expectedLine, r.readLine());
- }
- } catch (IOException ioe) {
- LOG.error("Got IOException verifying results: " + ioe.toString());
- ioe.printStackTrace();
- fail(ioe.toString());
- } finally {
- IOUtils.closeStream(r);
- }
- }
-
- private void runOracleTest(String [] expectedResults) throws IOException {
-
- Path warehousePath = new Path(this.getWarehouseDir());
- Path tablePath = new Path(warehousePath, TABLE_NAME);
- Path filePath = new Path(tablePath, "part-m-00000");
-
- File tableFile = new File(tablePath.toString());
- if (tableFile.exists() && tableFile.isDirectory()) {
- // remove the directory before running the import.
- FileListing.recursiveDeleteDir(tableFile);
- }
-
- String [] argv = getArgv();
- try {
- runImport(argv);
- } catch (IOException ioe) {
- LOG.error("Got IOException during import: " + ioe.toString());
- ioe.printStackTrace();
- fail(ioe.toString());
- }
-
- File f = new File(filePath.toString());
- assertTrue("Could not find imported data file", f.exists());
- BufferedReader r = null;
- try {
- // Read through the file and make sure it's all there.
- r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
- for (String expectedLine : expectedResults) {
- compareRecords(expectedLine, r.readLine());
- }
- } catch (IOException ioe) {
- LOG.error("Got IOException verifying results: " + ioe.toString());
- ioe.printStackTrace();
- fail(ioe.toString());
- } finally {
- IOUtils.closeStream(r);
- }
- }
-
- @Test
- public void testOracleImport() throws IOException {
- // no quoting of strings allowed. NOTE: Oracle JDBC 11.1 drivers
- // auto-cast SQL DATE to java.sql.Timestamp. Even if you define your
- // columns as DATE in Oracle, they may still contain time information, so
- // the JDBC drivers lie to us and will never tell us we have a strict DATE
- // type. Thus we include HH:MM:SS.mmmmm below.
- // See http://www.oracle.com
- // /technology/tech/java/sqlj_jdbc/htdocs/jdbc_faq.html#08_01
- String [] expectedResults = {
- "1,Aaron,2009-05-14 00:00:00.0,1000000,engineering,"
- + "2009-12-29 12:00:00.0,2009-12-29 12:00:00.0",
- "2,Bob,2009-04-20 00:00:00.0,400,sales,"
- + "2009-12-30 12:00:00.0,2009-12-30 12:00:00.0",
- "3,Fred,2009-01-23 00:00:00.0,15,marketing,"
- + "2009-12-31 12:00:00.0,2009-12-31 12:00:00.0",
- };
-
- runOracleTest(expectedResults);
- }
-
- @Test
- public void testSecondaryTableImport() throws IOException {
-
- String [] expectedResults = {
- "1,MercuryCorp",
- "2,VenusCorp",
- "3,EarthCorp",
- "4,MarsCorp",
- "5,JupiterCorp",
- "6,SaturnCorp",
- };
- runSecondaryTableTest(expectedResults);
- }
-
- /**
- * Compare two lines. Normalize the dates we receive based on the expected
- * time zone.
- * @param expectedLine expected line
- * @param receivedLine received line
- * @throws IOException exception during lines comparison
- */
- private void compareRecords(String expectedLine, String receivedLine)
- throws IOException {
- // handle null case
- if (expectedLine == null || receivedLine == null) {
- return;
- }
-
- // check if lines are equal
- if (expectedLine.equals(receivedLine)) {
- return;
- }
-
- // check if size is the same
- String [] expectedValues = expectedLine.split(",");
- String [] receivedValues = receivedLine.split(",");
- if (expectedValues.length != 7 || receivedValues.length != 7) {
- LOG.error("Number of expected fields did not match "
- + "number of received fields");
- throw new IOException("Number of expected fields did not match "
- + "number of received fields");
- }
-
- // check first 5 values
- boolean mismatch = false;
- for (int i = 0; !mismatch && i < 5; i++) {
- mismatch = !expectedValues[i].equals(receivedValues[i]);
- }
- if (mismatch) {
- throw new IOException("Expected:<" + expectedLine + "> but was:<"
- + receivedLine + ">");
- }
-
- Date expectedDate = null;
- Date receivedDate = null;
- DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S");
- int offset = TimeZone.getDefault().getOffset(System.currentTimeMillis())
- / 3600000;
- for (int i = 5; i < 7; i++) {
- // parse expected timestamp.
- try {
- expectedDate = df.parse(expectedValues[i]);
- } catch (ParseException ex) {
- LOG.error("Could not parse expected timestamp: " + expectedValues[i]);
- throw new IOException("Could not parse expected timestamp: "
- + expectedValues[i]);
- }
-
- // parse received timestamp
- try {
- receivedDate = df.parse(receivedValues[i]);
- } catch (ParseException ex) {
- LOG.error("Could not parse received timestamp: " + receivedValues[i]);
- throw new IOException("Could not parse received timestamp: "
- + receivedValues[i]);
- }
-
- // compare two timestamps considering timezone offset
- Calendar expectedCal = Calendar.getInstance();
- expectedCal.setTime(expectedDate);
- expectedCal.add(Calendar.HOUR, offset);
-
- Calendar receivedCal = Calendar.getInstance();
- receivedCal.setTime(receivedDate);
-
- if (!expectedCal.equals(receivedCal)) {
- throw new IOException("Expected:<" + expectedLine + "> but was:<"
- + receivedLine + ">, while timezone offset is: " + offset);
- }
- }
- }
-
- @Test
- public void testPurgeClosedConnections() throws Exception {
- // Ensure that after an Oracle ConnManager releases any connections
- // back into the cache (or closes them as redundant), it does not
- // attempt to re-use the closed connection.
-
- SqoopOptions options = new SqoopOptions(OracleUtils.CONNECT_STRING,
- TABLE_NAME);
- OracleUtils.setOracleAuth(options);
-
- // Create a connection manager, use it, and then recycle its connection
- // into the cache.
- ConnManager m1 = new OracleManager(options);
- Connection c1 = m1.getConnection();
- PreparedStatement s = c1.prepareStatement("SELECT 1 FROM dual",
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- ResultSet rs = null;
- try {
- rs = s.executeQuery();
- rs.close();
- } finally {
- s.close();
- }
-
- ConnManager m2 = new OracleManager(options);
- Connection c2 = m2.getConnection(); // get a new connection.
-
- m1.close(); // c1 should now be cached.
-
- // Use the second connection to run a statement.
- s = c2.prepareStatement("SELECT 2 FROM dual",
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- rs = s.executeQuery();
- rs.close();
- } finally {
- s.close();
- }
-
- m2.close(); // c2 should be discarded (c1 is already cached).
-
- // Try to get another connection from m2. This should result in
- // a completely different connection getting served back to us.
- Connection c2a = m2.getConnection();
-
- assertFalse(c1.isClosed());
- assertTrue(c2.isClosed());
- assertFalse(c2a.isClosed());
-
- s = c2a.prepareStatement("SELECT 3 FROM dual",
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- rs = s.executeQuery();
- rs.close();
- } finally {
- s.close();
- }
-
- m2.close(); // Close the manager's active connection again.
- }
-
- @Test
- public void testSessionUserName() throws Exception {
- SqoopOptions options = new SqoopOptions(OracleUtils.CONNECT_STRING,
- TABLE_NAME);
- OracleUtils.setOracleAuth(options);
-
- // Create a connection manager and get a connection
- OracleManager m1 = new OracleManager(options);
- Connection c1 = m1.getConnection();
- // Make sure that the session username is the same as the Oracle
- // sqoop user name
- String sessionUserName = m1.getSessionUser(c1);
- assertEquals(OracleUtils.ORACLE_USER_NAME, sessionUserName);
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/OracleUtils.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/OracleUtils.java b/src/test/com/cloudera/sqoop/manager/OracleUtils.java
deleted file mode 100644
index e99dfa9..0000000
--- a/src/test/com/cloudera/sqoop/manager/OracleUtils.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.manager;
-
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.cloudera.sqoop.SqoopOptions;
-
-/**
- * Helper methods for Oracle testing.
- */
-public final class OracleUtils {
-
- public static final Log LOG = LogFactory.getLog(OracleUtils.class.getName());
-
- // Express edition hardcoded name.
- public static final String ORACLE_DATABASE_NAME = "xe";
-
- public static final String CONNECT_STRING = System.getProperty("sqoop.test.oracle.connectstring", "jdbc:oracle:thin:@//localhost/" + ORACLE_DATABASE_NAME);
- public static final String ORACLE_USER_NAME = System.getProperty("sqoop.test.oracle.username", "SQOOPTEST");
- public static final String ORACLE_USER_PASS = System.getProperty("sqoop.test.oracle.password", "12345");
-
- public static final String ORACLE_SECONDARY_USER_NAME = "SQOOPTEST2";
- public static final String ORACLE_SECONDARY_USER_PASS = "ABCDEF";
-
- public static final String ORACLE_INVALID_USER_NAME = "invalidusr";
- public static final String SYSTEMTEST_TABLE_NAME = "ORAOOP_TEST";
- public static final int SYSTEMTEST_NUM_ROWS = 100;
- public static final int INTEGRATIONTEST_NUM_ROWS = 10000;
- // Number of mappers if wanting to override default setting
- public static final int NUM_MAPPERS = 0;
- // Oracle degree of parallelism to use when creating table.
- // If 0 we will calculate a recommended value
- public static final int ORACLE_PARALLEL_DEGREE = 0;
-
- private OracleUtils() { }
-
- public static void setOracleAuth(SqoopOptions options) {
- options.setUsername(ORACLE_USER_NAME);
- options.setPassword(ORACLE_USER_PASS);
- }
-
- public static void setOracleSecondaryUserAuth(SqoopOptions options) {
- options.setUsername(ORACLE_SECONDARY_USER_NAME);
- options.setPassword(ORACLE_SECONDARY_USER_PASS);
- }
-
- /**
- * Drop a table if it exists.
- */
- public static void dropTable(String tableName, ConnManager manager)
- throws SQLException {
- Connection connection = null;
- Statement st = null;
-
- try {
- connection = manager.getConnection();
- connection.setAutoCommit(false);
- st = connection.createStatement();
-
- // create the database table and populate it with data.
- st.executeUpdate(getDropTableStatement(tableName));
-
- connection.commit();
- } finally {
- try {
- if (null != st) {
- st.close();
- }
- } catch (SQLException sqlE) {
- LOG.warn("Got SQLException when closing connection: " + sqlE);
- }
- }
- }
-
- public static String getDropTableStatement(String tableName) {
- return "BEGIN EXECUTE IMMEDIATE 'DROP TABLE " + tableName + "'; "
- + "exception when others then null; end;";
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java b/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java
deleted file mode 100644
index da354bb..0000000
--- a/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.manager;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapred.JobConf;
-import org.junit.Test;
-
-import com.cloudera.sqoop.TestExport;
-import com.cloudera.sqoop.mapreduce.db.DBConfiguration;
-
-
-/**
- * Test the PGBulkloadManager implementations.
- * PGBulkloadManager uses both JDBC driver and pg_bulkload to facilitate it.
- *
- * Since this requires a Postgresql installation on your local machine to use,
- * this class is named in such a way that Hadoop's default QA process does not
- * run it.
- *
- * You need to run this manually with -Dtestcase=PGBulkloadManagerManualTest.
- *
- * You need to put Postgresql's JDBC driver library into lib dir.
- *
- * You need to create a sqooptest superuser and database and tablespace,
- * and install pg_bulkload for sqooptest database:
- *
- * $ sudo -u postgres createuser -U postgres -s sqooptest
- * $ sudo -u postgres createdb -U sqooptest sqooptest
- * $ sudo -u postgres mkdir /var/pgdata/stagingtablespace
- * $ psql -U sqooptest
- * -f /usr/local/share/postgresql/contrib/pg_bulkload.sql sqooptest
- * $ psql -U sqooptest sqooptest
- * sqooptest=# CREATE USER sqooptest;
- * sqooptest=# CREATE DATABASE sqooptest;
- * sqooptest=# CREATE TABLESPACE sqooptest
- * LOCATION '/var/pgdata/stagingtablespace';
- * sqooptest=# \q
- *
- */
-public class PGBulkloadManagerManualTest extends TestExport {
-
- public static final Log LOG =
- LogFactory.getLog(PGBulkloadManagerManualTest.class.getName());
- private DBConfiguration dbConf;
- static final String HOST_URL =
- System.getProperty("sqoop.test.postgresql.connectstring.host_url",
- "jdbc:postgresql://localhost/");
- static final String DATABASE =
- System.getProperty("sqoop.test.postgresql.database", "sqooptest");
- static final String TABLESPACE =
- System.getProperty("sqoop.test.postgresql.tablespace", "sqooptest");
- static final String USERNAME =
- System.getProperty("sqoop.test.postgresql.username", "sqooptest");
- static final String PG_BULKLOAD =
- System.getProperty("sqoop.test.postgresql.pg_bulkload", "pg_bulkload");
- static final String CONNECT_STRING = HOST_URL + DATABASE;
-
- public PGBulkloadManagerManualTest() {
- JobConf conf = new JobConf(getConf());
- DBConfiguration.configureDB(conf,
- "org.postgresql.Driver",
- getConnectString(),
- getUserName(),
- (String) null, (Integer) null);
- dbConf = new DBConfiguration(conf);
- }
-
-
- @Override
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
-
- @Override
- protected String getConnectString() {
- return CONNECT_STRING;
- }
-
-
- protected String getUserName() {
- return USERNAME;
- }
-
-
- @Override
- protected String getTablePrefix() {
- return super.getTablePrefix().toLowerCase();
- }
-
-
- @Override
- protected String getTableName() {
- return super.getTableName().toLowerCase();
- }
-
- @Override
- public String getStagingTableName() {
- return super.getStagingTableName().toLowerCase();
- }
-
-
- @Override
- protected Connection getConnection() {
- try {
- Connection conn = dbConf.getConnection();
- conn.setAutoCommit(false);
- PreparedStatement stmt =
- conn.prepareStatement("SET extra_float_digits TO 0");
- stmt.executeUpdate();
- conn.commit();
- return conn;
- } catch (SQLException sqlE) {
- LOG.error("Could not get connection to test server: " + sqlE);
- return null;
- } catch (ClassNotFoundException cnfE) {
- LOG.error("Could not find driver class: " + cnfE);
- return null;
- }
- }
-
-
- @Override
- protected String getDropTableStatement(String tableName) {
- return "DROP TABLE IF EXISTS " + tableName;
- }
-
-
- @Override
- protected String[] getArgv(boolean includeHadoopFlags,
- int rowsPerStatement,
- int statementsPerTx,
- String... additionalArgv) {
- ArrayList<String> args =
- new ArrayList<String>(Arrays.asList(additionalArgv));
- args.add("-D");
- args.add("pgbulkload.bin=" + PG_BULKLOAD);
- args.add("--username");
- args.add(getUserName());
- args.add("--connection-manager");
- args.add("org.apache.sqoop.manager.PGBulkloadManager");
- args.add("--staging-table");
- args.add("dummy");
- args.add("--clear-staging-table");
- return super.getArgv(includeHadoopFlags,
- rowsPerStatement,
- statementsPerTx,
- args.toArray(new String[0]));
- }
-
-
- @Override
- protected String [] getCodeGenArgv(String... extraArgs) {
- ArrayList<String> args = new ArrayList<String>(Arrays.asList(extraArgs));
- args.add("--username");
- args.add(getUserName());
- return super.getCodeGenArgv(args.toArray(new String[0]));
- }
-
-
- @Override
- public void testColumnsExport() throws IOException, SQLException {
- // PGBulkloadManager does not support --columns option.
- }
-
- @Test
- public void testMultiReduceExport() throws IOException, SQLException {
- multiFileTest(2, 10, 2, "-D", "mapred.reduce.tasks=2");
- }
-
- @Test
- public void testMultiReduceExportWithNewProp()
- throws IOException, SQLException {
- multiFileTest(2, 10, 2, "-D", "mapreduce.job.reduces=2");
- }
-
- @Test
- public void testExportWithTablespace() throws IOException, SQLException {
- multiFileTest(1, 10, 1,
- "-D", "pgbulkload.staging.tablespace=" + TABLESPACE);
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/PostgresqlExportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/PostgresqlExportTest.java b/src/test/com/cloudera/sqoop/manager/PostgresqlExportTest.java
deleted file mode 100644
index 3323d07..0000000
--- a/src/test/com/cloudera/sqoop/manager/PostgresqlExportTest.java
+++ /dev/null
@@ -1,490 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.sqoop.manager;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ExportJobTestCase;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.Writer;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- *
- */
-public class PostgresqlExportTest extends ExportJobTestCase {
- public static final Log LOG = LogFactory.getLog(
- PostgresqlExportTest.class.getName());
-
- static final String HOST_URL = System.getProperty(
- "sqoop.test.postgresql.connectstring.host_url",
- "jdbc:postgresql://localhost/");
- static final String DATABASE_USER = System.getProperty(
- "sqoop.test.postgresql.username",
- "sqooptest");
- static final String DATABASE_NAME = System.getProperty(
- "sqoop.test.postgresql.database",
- "sqooptest");
- static final String PASSWORD = System.getProperty(
- "sqoop.test.postgresql.password");
-
- static final String TABLE_NAME = "EMPLOYEES_PG";
- static final String PROCEDURE_NAME = "INSERT_AN_EMPLOYEE";
- static final String STAGING_TABLE_NAME = "STAGING";
- static final String SCHEMA_PUBLIC = "public";
- static final String SCHEMA_SPECIAL = "special";
- static final String CONNECT_STRING = HOST_URL + DATABASE_NAME;
-
- protected Connection connection;
-
- @Override
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- private String getDropTableStatement(String tableName, String schema) {
- return "DROP TABLE IF EXISTS " + quoteTableOrSchemaName(schema) + "." + quoteTableOrSchemaName(tableName);
- }
-
- @Before
- public void setUp() {
- super.setUp();
-
- LOG.debug("Setting up postgresql test: " + CONNECT_STRING);
-
- try {
- connection = DriverManager.getConnection(CONNECT_STRING, DATABASE_USER, PASSWORD);
- connection.setAutoCommit(false);
- } catch (SQLException ex) {
- LOG.error("Can't create connection", ex);
- throw new RuntimeException(ex);
- }
-
- createTable(TABLE_NAME, SCHEMA_PUBLIC);
- createTable(STAGING_TABLE_NAME, SCHEMA_PUBLIC);
- createTable(TABLE_NAME, SCHEMA_SPECIAL);
- createTable(STAGING_TABLE_NAME, SCHEMA_SPECIAL);
- createProcedure(PROCEDURE_NAME, SCHEMA_PUBLIC);
-
- LOG.debug("setUp complete.");
- }
-
- @Override
- public void tearDown() {
- try {
- Statement stmt = connection.createStatement();
- stmt.executeUpdate(getDropTableStatement(TABLE_NAME, SCHEMA_PUBLIC));
- stmt.executeUpdate(getDropTableStatement(STAGING_TABLE_NAME, SCHEMA_PUBLIC));
- stmt.executeUpdate(getDropTableStatement(TABLE_NAME, SCHEMA_SPECIAL));
- stmt.executeUpdate(getDropTableStatement(STAGING_TABLE_NAME, SCHEMA_SPECIAL));
- } catch(SQLException e) {
- LOG.error("Can't clean up the database:", e);
- }
-
- super.tearDown();
-
- try {
- connection.close();
- } catch (SQLException e) {
- LOG.error("Ignoring exception in tearDown", e);
- }
- }
-
- private interface CreateIt {
- void createIt(
- Statement st,
- String fullName,
- ConnManager manager) throws SQLException;
- }
-
- private void createTable(String tableName, String schema) {
- CreateIt createIt = new CreateIt() {
- @Override
- public void createIt(
- Statement st,
- String fullName,
- ConnManager manager) throws SQLException {
- st.executeUpdate("CREATE TABLE " + fullName + " ("
- + manager.escapeColName("id") + " INT NOT NULL PRIMARY KEY, "
- + manager.escapeColName("name") + " VARCHAR(24) NOT NULL, "
- + manager.escapeColName("start_date") + " DATE, "
- + manager.escapeColName("salary") + " FLOAT, "
- + manager.escapeColName("dept") + " VARCHAR(32))");
- }
- };
- create(tableName, "TABLE", schema, createIt);
- }
-
- private void createProcedure(String procedureName, String schema) {
- CreateIt createIt = new CreateIt() {
- @Override
- public void createIt(
- Statement st,
- String fullName,
- ConnManager manager) throws SQLException {
- st.executeUpdate("CREATE OR REPLACE FUNCTION " + fullName + " ("
- + "IN " + manager.escapeColName("id") + " INT,"
- + "IN " + manager.escapeColName("name") + " VARCHAR(24),"
- + "IN " + manager.escapeColName("start_date") + " DATE,"
- + "IN " + manager.escapeColName("salary") + " FLOAT,"
- + "IN " + manager.escapeColName("dept") + " VARCHAR(32)"
- + ") "
- + "RETURNS VOID "
- + "AS $$ "
- + "BEGIN "
- + "INSERT INTO "
- + quoteTableOrSchemaName(SCHEMA_PUBLIC)
- + "."
- + quoteTableOrSchemaName(TABLE_NAME)
- + " ("
- + manager.escapeColName("id")
- +", "
- + manager.escapeColName("name")
- +", "
- + manager.escapeColName("start_date")
- +", "
- + manager.escapeColName("salary")
- +", "
- + manager.escapeColName("dept")
- + ") VALUES ("
- + manager.escapeColName("id")
- +", "
- + manager.escapeColName("name")
- +", "
- + manager.escapeColName("start_date")
- +", "
- + manager.escapeColName("salary")
- +", "
- + manager.escapeColName("dept")
- + ");"
- + "END;"
- + "$$ LANGUAGE plpgsql;");
- }
- };
- create(procedureName, "FUNCTION", schema, createIt);
- }
-
- private void create(
- String name,
- String type,
- String schema,
- CreateIt createIt) {
- SqoopOptions options = new SqoopOptions(CONNECT_STRING, name);
- options.setUsername(DATABASE_USER);
-
- ConnManager manager = null;
- Statement st = null;
-
- try {
- manager = new PostgresqlManager(options);
- st = connection.createStatement();
-
- // Create schema if not exists in dummy way (always create and ignore
- // errors.
- try {
- st.executeUpdate("CREATE SCHEMA " + quoteTableOrSchemaName(schema));
- connection.commit();
- } catch (SQLException e) {
- LOG.info("Couldn't create schema " + schema + " (is o.k. as long as"
- + "the schema already exists.", e);
- connection.rollback();
- }
-
- String fullTableName = quoteTableOrSchemaName(schema)
- + "." + quoteTableOrSchemaName(name);
-
- try {
- // Try to remove the table first. DROP TABLE IF EXISTS didn't
- // get added until pg 8.3, so we just use "DROP TABLE" and ignore
- // any exception here if one occurs.
- st.executeUpdate("DROP " + type + " " + fullTableName);
- } catch (SQLException e) {
- LOG.info("Couldn't drop "
- + type.toLowerCase()
- + " " +fullTableName
- + " (ok)",
- e);
- // Now we need to reset the transaction.
- connection.rollback();
- }
-
- createIt.createIt(st, fullTableName, manager);
-
- connection.commit();
- } catch (SQLException sqlE) {
- LOG.error("Encountered SQL Exception: " + sqlE);
- sqlE.printStackTrace();
- fail("SQLException when running test setUp(): " + sqlE);
- } finally {
- try {
- if (null != st) {
- st.close();
- }
-
- if (null != manager) {
- manager.close();
- }
- } catch (SQLException sqlE) {
- LOG.warn("Got SQLException when closing connection: " + sqlE);
- }
- }
-
- LOG.debug("setUp complete.");
- }
-
- private String [] getArgv(boolean useTable,
- String... extraArgs) {
- ArrayList<String> args = new ArrayList<String>();
-
- CommonArgs.addHadoopFlags(args);
-
- if (useTable) {
- args.add("--table");
- args.add(TABLE_NAME);
- } else {
- args.add("--call");
- args.add(PROCEDURE_NAME);
- }
- args.add("--export-dir");
- args.add(getWarehouseDir());
- args.add("--fields-terminated-by");
- args.add(",");
- args.add("--lines-terminated-by");
- args.add("\\n");
- args.add("--connect");
- args.add(CONNECT_STRING);
- args.add("--username");
- args.add(DATABASE_USER);
- args.add("--password");
- args.add(PASSWORD);
- args.add("-m");
- args.add("1");
-
- for (String arg : extraArgs) {
- args.add(arg);
- }
-
- return args.toArray(new String[0]);
- }
-
- protected void createTestFile(String filename,
- String[] lines)
- throws IOException {
- new File(getWarehouseDir()).mkdirs();
- File file = new File(getWarehouseDir() + "/" + filename);
- Writer output = new BufferedWriter(new FileWriter(file));
- for(String line : lines) {
- output.write(line);
- output.write("\n");
- }
- output.close();
- }
-
- @Test
- public void testExport() throws IOException, SQLException {
- createTestFile("inputFile", new String[] {
- "2,Bob,2009-04-20,400,sales",
- "3,Fred,2009-01-23,15,marketing",
- });
-
- runExport(getArgv(true));
-
- assertRowCount(2, quoteTableOrSchemaName(TABLE_NAME), connection);
- }
-
- @Test
- public void testExportUsingProcedure() throws IOException, SQLException {
- createTestFile("inputFile", new String[] {
- "2,Bob,2009-04-20,400,sales",
- "3,Fred,2009-01-23,15,marketing",
- });
-
- runExport(getArgv(false));
-
- assertRowCount(2, quoteTableOrSchemaName(TABLE_NAME), connection);
- }
-
- @Test
- public void testExportStaging() throws IOException, SQLException {
- createTestFile("inputFile", new String[] {
- "2,Bob,2009-04-20,400,sales",
- "3,Fred,2009-01-23,15,marketing",
- });
-
- String[] extra = new String[] {"--staging-table", STAGING_TABLE_NAME, };
-
- runExport(getArgv(true, extra));
-
- assertRowCount(2, quoteTableOrSchemaName(TABLE_NAME), connection);
- }
-
- @Test
- public void testExportDirect() throws IOException, SQLException {
- createTestFile("inputFile", new String[] {
- "2,Bob,2009-04-20,400,sales",
- "3,Fred,2009-01-23,15,marketing",
- });
-
- String[] extra = new String[] {"--direct"};
-
- runExport(getArgv(true, extra));
-
- assertRowCount(2, quoteTableOrSchemaName(TABLE_NAME), connection);
- }
-
- @Test
- public void testExportCustomSchema() throws IOException, SQLException {
- createTestFile("inputFile", new String[] {
- "2,Bob,2009-04-20,400,sales",
- "3,Fred,2009-01-23,15,marketing",
- });
-
- String[] extra = new String[] {"--",
- "--schema",
- SCHEMA_SPECIAL,
- };
-
- runExport(getArgv(true, extra));
-
- assertRowCount(2,
- quoteTableOrSchemaName(SCHEMA_SPECIAL)
- + "." + quoteTableOrSchemaName(TABLE_NAME),
- connection);
- }
-
- @Test
- public void testExportCustomSchemaStaging() throws IOException, SQLException {
- createTestFile("inputFile", new String[] {
- "2,Bob,2009-04-20,400,sales",
- "3,Fred,2009-01-23,15,marketing",
- });
-
- String[] extra = new String[] {
- "--staging-table",
- STAGING_TABLE_NAME,
- "--",
- "--schema",
- SCHEMA_SPECIAL,
- };
-
- runExport(getArgv(true, extra));
-
- assertRowCount(2,
- quoteTableOrSchemaName(SCHEMA_SPECIAL)
- + "." + quoteTableOrSchemaName(TABLE_NAME),
- connection);
- }
-
- @Test
- public void testExportCustomSchemaStagingClear()
- throws IOException, SQLException {
- createTestFile("inputFile", new String[] {
- "2,Bob,2009-04-20,400,sales",
- "3,Fred,2009-01-23,15,marketing",
- });
-
- String[] extra = new String[] {
- "--staging-table",
- STAGING_TABLE_NAME,
- "--clear-staging-table",
- "--",
- "--schema",
- SCHEMA_SPECIAL,
- };
-
- runExport(getArgv(true, extra));
-
- assertRowCount(2,
- quoteTableOrSchemaName(SCHEMA_SPECIAL)
- + "." + quoteTableOrSchemaName(TABLE_NAME),
- connection);
- }
-
- @Test
- public void testExportCustomSchemaDirect() throws IOException, SQLException {
- createTestFile("inputFile", new String[] {
- "2,Bob,2009-04-20,400,sales",
- "3,Fred,2009-01-23,15,marketing",
- });
-
- String[] extra = new String[] {
- "--direct",
- "--",
- "--schema",
- SCHEMA_SPECIAL,
- };
-
- runExport(getArgv(true, extra));
-
- assertRowCount(2,
- quoteTableOrSchemaName(SCHEMA_SPECIAL)
- + "." + quoteTableOrSchemaName(TABLE_NAME),
- connection);
- }
-
- public static void assertRowCount(long expected,
- String tableName,
- Connection connection) {
- Statement stmt = null;
- ResultSet rs = null;
- try {
- stmt = connection.createStatement();
- rs = stmt.executeQuery("SELECT count(*) FROM " + tableName);
-
- rs.next();
-
- assertEquals(expected, rs.getLong(1));
- } catch (SQLException e) {
- LOG.error("Can't verify number of rows", e);
- fail();
- } finally {
- try {
- connection.commit();
-
- if (stmt != null) {
- stmt.close();
- }
- if (rs != null) {
- rs.close();
- }
- } catch (SQLException ex) {
- LOG.info("Ignored exception in finally block.");
- }
- }
- }
-
- public String quoteTableOrSchemaName(String tableName) {
- return "\"" + tableName + "\"";
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/PostgresqlExternalTableImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/PostgresqlExternalTableImportTest.java b/src/test/com/cloudera/sqoop/manager/PostgresqlExternalTableImportTest.java
deleted file mode 100644
index 29b3fdd..0000000
--- a/src/test/com/cloudera/sqoop/manager/PostgresqlExternalTableImportTest.java
+++ /dev/null
@@ -1,285 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.cloudera.sqoop.manager;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
-import com.cloudera.sqoop.util.FileListing;
-
-public class PostgresqlExternalTableImportTest extends ImportJobTestCase {
-
- public static final Log LOG = LogFactory
- .getLog(PostgresqlExternalTableImportTest.class.getName());
- static final String HOST_URL = System.getProperty("sqoop.test.postgresql.connectstring.host_url",
- "jdbc:postgresql://localhost/");
- static final String DATABASE_USER = System.getProperty(
- "sqoop.test.postgresql.username", "sqooptest");
- static final String DATABASE_NAME = System.getProperty(
- "sqoop.test.postgresql.database", "sqooptest");
- static final String PASSWORD = System.getProperty("sqoop.test.postgresql.password");
-
- static final String TABLE_NAME = "EMPLOYEES_PG";
- static final String NULL_TABLE_NAME = "NULL_EMPLOYEES_PG";
- static final String SPECIAL_TABLE_NAME = "EMPLOYEES_PG's";
- static final String DIFFERENT_TABLE_NAME = "DIFFERENT_TABLE";
- static final String SCHEMA_PUBLIC = "public";
- static final String SCHEMA_SPECIAL = "special";
- static final String CONNECT_STRING = HOST_URL + DATABASE_NAME;
- static final String EXTERNAL_TABLE_DIR = "/tmp/external/employees_pg";
- protected Connection connection;
-
- @Override
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- public String quoteTableOrSchemaName(String tableName) {
- return "\"" + tableName + "\"";
- }
-
- private String getDropTableStatement(String tableName, String schema) {
- return "DROP TABLE IF EXISTS " + quoteTableOrSchemaName(schema) + "."
- + quoteTableOrSchemaName(tableName);
- }
-
- @Before
- public void setUp() {
- super.setUp();
-
- LOG.debug("Setting up another postgresql test: " + CONNECT_STRING);
-
- setUpData(TABLE_NAME, SCHEMA_PUBLIC, false);
- setUpData(NULL_TABLE_NAME, SCHEMA_PUBLIC, true);
- setUpData(SPECIAL_TABLE_NAME, SCHEMA_PUBLIC, false);
- setUpData(DIFFERENT_TABLE_NAME, SCHEMA_SPECIAL, false);
-
- LOG.debug("setUp complete.");
- }
-
- @After
- public void tearDown() {
- try {
- Statement stmt = connection.createStatement();
- stmt.executeUpdate(getDropTableStatement(TABLE_NAME, SCHEMA_PUBLIC));
- stmt.executeUpdate(getDropTableStatement(NULL_TABLE_NAME, SCHEMA_PUBLIC));
- stmt.executeUpdate(getDropTableStatement(SPECIAL_TABLE_NAME, SCHEMA_PUBLIC));
- stmt.executeUpdate(getDropTableStatement(DIFFERENT_TABLE_NAME, SCHEMA_SPECIAL));
- } catch (SQLException e) {
- LOG.error("Can't clean up the database:", e);
- }
-
- super.tearDown();
-
- try {
- connection.close();
- } catch (SQLException e) {
- LOG.error("Ignoring exception in tearDown", e);
- }
- }
-
- public void setUpData(String tableName, String schema, boolean nullEntry) {
- SqoopOptions options = new SqoopOptions(CONNECT_STRING, tableName);
- options.setUsername(DATABASE_USER);
- options.setPassword(PASSWORD);
-
- ConnManager manager = null;
- Statement st = null;
-
- try {
- manager = new PostgresqlManager(options);
- connection = manager.getConnection();
- connection.setAutoCommit(false);
- st = connection.createStatement();
-
- // Create schema if not exists in dummy way (always create and ignore
- // errors.
- try {
- st.executeUpdate("CREATE SCHEMA " + manager.escapeTableName(schema));
- connection.commit();
- } catch (SQLException e) {
- LOG.info("Couldn't create schema " + schema + " (is o.k. as long as"
- + "the schema already exists.");
- connection.rollback();
- }
-
- String fullTableName = manager.escapeTableName(schema) + "."
- + manager.escapeTableName(tableName);
- LOG.info("Creating table: " + fullTableName);
-
- try {
- // Try to remove the table first. DROP TABLE IF EXISTS didn't
- // get added until pg 8.3, so we just use "DROP TABLE" and ignore
- // any exception here if one occurs.
- st.executeUpdate("DROP TABLE " + fullTableName);
- } catch (SQLException e) {
- LOG.info("Couldn't drop table " + schema + "." + tableName + " (ok)");
- // Now we need to reset the transaction.
- connection.rollback();
- }
-
- st.executeUpdate("CREATE TABLE " + fullTableName + " (" + manager.escapeColName("id")
- + " INT NOT NULL PRIMARY KEY, " + manager.escapeColName("name")
- + " VARCHAR(24) NOT NULL, " + manager.escapeColName("start_date") + " DATE, "
- + manager.escapeColName("Salary") + " FLOAT, " + manager.escapeColName("Fired")
- + " BOOL, " + manager.escapeColName("dept") + " VARCHAR(32))");
-
- st.executeUpdate("INSERT INTO " + fullTableName
- + " VALUES(1,'Aaron','2009-05-14',1000000.00,TRUE,'engineering')");
- st.executeUpdate("INSERT INTO " + fullTableName
- + " VALUES(2,'Bob','2009-04-20',400.00,TRUE,'sales')");
- st.executeUpdate("INSERT INTO " + fullTableName
- + " VALUES(3,'Fred','2009-01-23',15.00,FALSE,'marketing')");
- if (nullEntry) {
- st.executeUpdate("INSERT INTO " + fullTableName + " VALUES(4,'Mike',NULL,NULL,NULL,NULL)");
-
- }
- connection.commit();
- } catch (SQLException sqlE) {
- LOG.error("Encountered SQL Exception: " + sqlE);
- sqlE.printStackTrace();
- fail("SQLException when running test setUp(): " + sqlE);
- } finally {
- try {
- if (null != st) {
- st.close();
- }
-
- if (null != manager) {
- manager.close();
- }
- } catch (SQLException sqlE) {
- LOG.warn("Got SQLException when closing connection: " + sqlE);
- }
- }
-
- LOG.debug("setUp complete.");
- }
-
- private String[] getArgv(boolean isDirect, String tableName, String... extraArgs) {
- ArrayList<String> args = new ArrayList<String>();
-
- CommonArgs.addHadoopFlags(args);
-
- args.add("--table");
- args.add(tableName);
- args.add("--external-table-dir");
- args.add(EXTERNAL_TABLE_DIR);
- args.add("--hive-import");
- args.add("--warehouse-dir");
- args.add(getWarehouseDir());
- args.add("--connect");
- args.add(CONNECT_STRING);
- args.add("--username");
- args.add(DATABASE_USER);
- args.add("--password");
- args.add(PASSWORD);
- args.add("--where");
- args.add("id > 1");
- args.add("-m");
- args.add("1");
-
- if (isDirect) {
- args.add("--direct");
- }
-
- for (String arg : extraArgs) {
- args.add(arg);
- }
-
- return args.toArray(new String[0]);
- }
-
- private void doImportAndVerify(boolean isDirect, String[] expectedResults, String tableName,
- String... extraArgs) throws IOException {
-
- Path tablePath = new Path(EXTERNAL_TABLE_DIR);
-
- // if importing with merge step, directory should exist and output should be
- // from a reducer
- boolean isMerge = Arrays.asList(extraArgs).contains("--merge-key");
- Path filePath = new Path(tablePath, isMerge ? "part-r-00000" : "part-m-00000");
-
- File tableFile = new File(tablePath.toString());
- if (tableFile.exists() && tableFile.isDirectory() && !isMerge) {
- // remove the directory before running the import.
- FileListing.recursiveDeleteDir(tableFile);
- }
-
- String[] argv = getArgv(isDirect, tableName, extraArgs);
- try {
- runImport(argv);
- } catch (IOException ioe) {
- LOG.error("Got IOException during import: " + ioe.toString());
- ioe.printStackTrace();
- fail(ioe.toString());
- }
-
- File f = new File(filePath.toString());
- assertTrue("Could not find imported data file, " + f, f.exists());
- BufferedReader r = null;
- try {
- // Read through the file and make sure it's all there.
- r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
- for (String expectedLine : expectedResults) {
- assertEquals(expectedLine, r.readLine());
- }
- } catch (IOException ioe) {
- LOG.error("Got IOException verifying results: " + ioe.toString());
- ioe.printStackTrace();
- fail(ioe.toString());
- } finally {
- IOUtils.closeStream(r);
- }
- }
-
- @Test
- public void testJdbcBasedImport() throws IOException {
- // separator is different to other tests
- // because the CREATE EXTERNAL TABLE DDL is
- // ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
- char sep = '\001';
- String[] expectedResults = {
- "2" + sep + "Bob" + sep + "2009-04-20" + sep + "400.0" + sep + "true" + sep + "sales",
- "3" + sep + "Fred" + sep + "2009-01-23" + sep + "15.0" + sep + "false" + sep + "marketing" };
- doImportAndVerify(false, expectedResults, TABLE_NAME);
- }
-}