You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ma...@apache.org on 2017/07/21 11:29:52 UTC
[1/3] sqoop git commit: SQOOP-3198: Fix DirectMySQLExportTest and
OracleExportTest
Repository: sqoop
Updated Branches:
refs/heads/branch-1.4.7 f6f59892c -> 13ec21cda
SQOOP-3198: Fix DirectMySQLExportTest and OracleExportTest
(Szabolcs Vasas via Anna Szonyi)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/53cedb93
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/53cedb93
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/53cedb93
Branch: refs/heads/branch-1.4.7
Commit: 53cedb93b20f160ea3d84ab389fa289ae8120d54
Parents: f6f5989
Author: Anna Szonyi <an...@apache.org>
Authored: Tue Jun 20 10:28:01 2017 +0200
Committer: Attila Szabo <ma...@apache.org>
Committed: Fri Jul 21 11:28:13 2017 +0200
----------------------------------------------------------------------
src/test/com/cloudera/sqoop/TestExport.java | 37 ++++++++++----------
.../sqoop/manager/DirectMySQLExportTest.java | 17 +++++++--
.../apache/sqoop/TestExportUsingProcedure.java | 4 +--
3 files changed, 35 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/53cedb93/src/test/com/cloudera/sqoop/TestExport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestExport.java b/src/test/com/cloudera/sqoop/TestExport.java
index 8b468c8..7948dae 100644
--- a/src/test/com/cloudera/sqoop/TestExport.java
+++ b/src/test/com/cloudera/sqoop/TestExport.java
@@ -124,6 +124,20 @@ public class TestExport extends ExportJobTestCase {
String getType();
}
+ protected static class IntColumnGenerator implements ColumnGenerator {
+ public String getExportText(int rowNum) {
+ int day = rowNum + 1;
+ return String.valueOf(day);
+ }
+ public String getVerifyText(int rowNum) {
+ int day = rowNum + 1;
+ return String.valueOf(day);
+ }
+ public String getType() {
+ return "INT";
+ }
+ }
+
/**
* Create a data file that gets exported to the db.
* @param fileNum the number of the file (for multi-file export)
@@ -906,14 +920,14 @@ public class TestExport extends ExportJobTestCase {
public void testLessColumnsInFileThanInTable() throws IOException, SQLException {
final int TOTAL_RECORDS = 10;
- ColumnGenerator genDate = getDateColumnGenerator();
+ ColumnGenerator genInteger = new IntColumnGenerator();
ColumnGenerator genTime = getTimeColumnGenerator();
- createTextFile(0, TOTAL_RECORDS, false, genDate);
- createTable(genDate, genTime);
+ createTextFile(0, TOTAL_RECORDS, false, genInteger);
+ createTable(genInteger, genTime);
runExport(getArgv(true, 10, 10));
verifyExport(TOTAL_RECORDS);
- assertColMinAndMax(forIdx(0), genDate);
+ assertColMinAndMax(forIdx(0), genInteger);
// test that the Time column is with NULL values
class NullColumnGenerator implements ColumnGenerator {
@@ -1045,21 +1059,6 @@ public class TestExport extends ExportJobTestCase {
}
}
- // a normal string column
- class IntColumnGenerator implements ColumnGenerator {
- public String getExportText(int rowNum) {
- int day = rowNum + 1;
- return String.valueOf(day);
- }
- public String getVerifyText(int rowNum) {
- int day = rowNum + 1;
- return String.valueOf(day);
- }
- public String getType() {
- return "INT";
- }
- }
-
// test that the second column is with NULL values after import
class NullColumnGenerator implements ColumnGenerator {
public String getExportText(int rowNum) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/53cedb93/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java b/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java
index ec7b84a..9fa8816 100644
--- a/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java
+++ b/src/test/com/cloudera/sqoop/manager/DirectMySQLExportTest.java
@@ -38,6 +38,7 @@ import org.junit.Before;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.TestExport;
import com.cloudera.sqoop.mapreduce.MySQLExportMapper;
+import org.junit.Ignore;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
@@ -302,18 +303,30 @@ public class DirectMySQLExportTest extends TestExport {
}
}
+ @Ignore("Ignoring this test as staging is not supported in direct mode.")
@Override
@Test
public void testMultiMapTextExportWithStaging()
throws IOException, SQLException {
- // disable this test as staging is not supported in direct mode
}
+ @Ignore("Ignoring this test as staging is not supported in direct mode.")
@Override
@Test
public void testMultiTransactionWithStaging()
throws IOException, SQLException {
- // disable this test as staging is not supported in direct mode
+ }
+
+ @Ignore("Ignoring this test as --input-null-non-string is not supported in direct mode.")
+ @Override
+ @Test
+ public void testLessColumnsInFileThanInTableInputNullIntPassed() throws IOException, SQLException {
+ }
+
+ @Ignore("Ignoring this test as --input-null-string is not supported in direct mode.")
+ @Override
+ @Test
+ public void testLessColumnsInFileThanInTableInputNullStringPassed() throws IOException, SQLException {
}
private void verifyTableColumnContents(Connection connection,
http://git-wip-us.apache.org/repos/asf/sqoop/blob/53cedb93/src/test/org/apache/sqoop/TestExportUsingProcedure.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/TestExportUsingProcedure.java b/src/test/org/apache/sqoop/TestExportUsingProcedure.java
index 68e79f1..1783676 100644
--- a/src/test/org/apache/sqoop/TestExportUsingProcedure.java
+++ b/src/test/org/apache/sqoop/TestExportUsingProcedure.java
@@ -343,11 +343,11 @@ public class TestExportUsingProcedure extends TestExport {
* columns in the table.
*/
public static void insertFunctiontestLessColumnsInFileThanInTable(int id, String msg,
- final Date date1, final Timestamp t2) throws SQLException {
+ final int number, final Timestamp t2) throws SQLException {
insertFunction(id, msg, new SetExtraArgs() {
@Override
public void set(PreparedStatement on) throws SQLException {
- on.setDate(3, date1);
+ on.setInt(3, number);
on.setNull(4, Types.TIMESTAMP);
}
});
[2/3] sqoop git commit: SQOOP-3197: Database credentials cannot be
configured in PostgreSQL third party tests
Posted by ma...@apache.org.
SQOOP-3197: Database credentials cannot be configured in PostgreSQL third party tests
(Szabolcs Vasas via Anna Szonyi)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/33be8445
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/33be8445
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/33be8445
Branch: refs/heads/branch-1.4.7
Commit: 33be84452a7a175bd4762fb68ef3d72d55c42412
Parents: 53cedb9
Author: Anna Szonyi <an...@apache.org>
Authored: Thu Jul 6 16:44:05 2017 +0200
Committer: Attila Szabo <ma...@apache.org>
Committed: Fri Jul 21 11:31:17 2017 +0200
----------------------------------------------------------------------
build.xml | 6 ++++
.../DirectPostgreSQLExportManualTest.java | 33 ++++++++++++++++++--
.../sqoop/manager/PostgresqlExportTest.java | 8 +++--
.../PostgresqlExternalTableImportTest.java | 8 +++--
.../sqoop/manager/PostgresqlImportTest.java | 9 ++++--
5 files changed, 53 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/33be8445/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 110cfa5..ede5ba8 100644
--- a/build.xml
+++ b/build.xml
@@ -218,6 +218,9 @@
<property name="sqoop.test.postgresql.username"
value="sqooptest" />
+ <property name="sqoop.test.postgresql.password"
+ value="" />
+
<property name="sqoop.test.postgresql.pg_bulkload"
value="pg_bulkload" />
@@ -847,6 +850,9 @@
<sysproperty key="sqoop.test.postgresql.username"
value="${sqoop.test.postgresql.username}" />
+ <sysproperty key="sqoop.test.postgresql.password"
+ value="${sqoop.test.postgresql.password}" />
+
<sysproperty key="sqoop.test.postgresql.pg_bulkload"
value="${sqoop.test.postgresql.pg_bulkload}" />
http://git-wip-us.apache.org/repos/asf/sqoop/blob/33be8445/src/test/com/cloudera/sqoop/manager/DirectPostgreSQLExportManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/DirectPostgreSQLExportManualTest.java b/src/test/com/cloudera/sqoop/manager/DirectPostgreSQLExportManualTest.java
index 52095ef..b7c7416 100644
--- a/src/test/com/cloudera/sqoop/manager/DirectPostgreSQLExportManualTest.java
+++ b/src/test/com/cloudera/sqoop/manager/DirectPostgreSQLExportManualTest.java
@@ -29,6 +29,8 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.JobConf;
import com.cloudera.sqoop.TestExport;
import com.cloudera.sqoop.mapreduce.db.DBConfiguration;
+import org.junit.Ignore;
+import org.junit.Test;
/**
@@ -64,6 +66,8 @@ public class DirectPostgreSQLExportManualTest extends TestExport {
System.getProperty("sqoop.test.postgresql.database", "sqooptest");
static final String USERNAME =
System.getProperty("sqoop.test.postgresql.username", "sqooptest");
+ static final String PASSWORD = System.getProperty(
+ "sqoop.test.postgresql.password");
static final String CONNECT_STRING = HOST_URL + DATABASE;
public DirectPostgreSQLExportManualTest() {
@@ -72,7 +76,7 @@ public class DirectPostgreSQLExportManualTest extends TestExport {
"org.postgresql.Driver",
getConnectString(),
getUserName(),
- (String) null, (Integer) null);
+ PASSWORD, (Integer) null);
dbConf = new DBConfiguration(conf);
}
@@ -138,6 +142,8 @@ public class DirectPostgreSQLExportManualTest extends TestExport {
new ArrayList<String>(Arrays.asList(additionalArgv));
args.add("--username");
args.add(getUserName());
+ args.add("--password");
+ args.add(PASSWORD);
args.add("--direct");
return super.getArgv(includeHadoopFlags,
rowsPerStatement,
@@ -150,11 +156,34 @@ public class DirectPostgreSQLExportManualTest extends TestExport {
ArrayList<String> args = new ArrayList<String>(Arrays.asList(extraArgs));
args.add("--username");
args.add(getUserName());
+ args.add("--password");
+ args.add(PASSWORD);
return super.getCodeGenArgv(args.toArray(new String[0]));
}
+ @Ignore("Ignoring this test case as direct export does not support --columns option.")
@Override
+ @Test
public void testColumnsExport() throws IOException, SQLException {
- // Direct export does not support --columns option.
}
+
+ @Ignore("Ignoring this test case as the scenario is not supported with direct export.")
+ @Override
+ @Test
+ public void testLessColumnsInFileThanInTable() throws IOException, SQLException {
+ }
+
+ @Ignore("Ignoring this test case as the scenario is not supported with direct export.")
+ @Override
+ @Test
+ public void testLessColumnsInFileThanInTableInputNullIntPassed() throws IOException, SQLException {
+ }
+
+ @Ignore("Ignoring this test case as the scenario is not supported with direct export.")
+ @Override
+ @Test
+ public void testLessColumnsInFileThanInTableInputNullStringPassed() throws IOException, SQLException {
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/33be8445/src/test/com/cloudera/sqoop/manager/PostgresqlExportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/PostgresqlExportTest.java b/src/test/com/cloudera/sqoop/manager/PostgresqlExportTest.java
index 38d6ba6..3323d07 100644
--- a/src/test/com/cloudera/sqoop/manager/PostgresqlExportTest.java
+++ b/src/test/com/cloudera/sqoop/manager/PostgresqlExportTest.java
@@ -51,13 +51,13 @@ public class PostgresqlExportTest extends ExportJobTestCase {
"sqoop.test.postgresql.connectstring.host_url",
"jdbc:postgresql://localhost/");
static final String DATABASE_USER = System.getProperty(
- "sqoop.test.postgresql.connectstring.username",
+ "sqoop.test.postgresql.username",
"sqooptest");
static final String DATABASE_NAME = System.getProperty(
- "sqoop.test.postgresql.connectstring.database",
+ "sqoop.test.postgresql.database",
"sqooptest");
static final String PASSWORD = System.getProperty(
- "sqoop.test.postgresql.connectstring.password");
+ "sqoop.test.postgresql.password");
static final String TABLE_NAME = "EMPLOYEES_PG";
static final String PROCEDURE_NAME = "INSERT_AN_EMPLOYEE";
@@ -286,6 +286,8 @@ public class PostgresqlExportTest extends ExportJobTestCase {
args.add(CONNECT_STRING);
args.add("--username");
args.add(DATABASE_USER);
+ args.add("--password");
+ args.add(PASSWORD);
args.add("-m");
args.add("1");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/33be8445/src/test/com/cloudera/sqoop/manager/PostgresqlExternalTableImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/PostgresqlExternalTableImportTest.java b/src/test/com/cloudera/sqoop/manager/PostgresqlExternalTableImportTest.java
index 0515e09..6403b32 100644
--- a/src/test/com/cloudera/sqoop/manager/PostgresqlExternalTableImportTest.java
+++ b/src/test/com/cloudera/sqoop/manager/PostgresqlExternalTableImportTest.java
@@ -35,10 +35,10 @@ public class PostgresqlExternalTableImportTest extends ImportJobTestCase {
static final String HOST_URL = System.getProperty("sqoop.test.postgresql.connectstring.host_url",
"jdbc:postgresql://localhost/");
static final String DATABASE_USER = System.getProperty(
- "sqoop.test.postgresql.connectstring.username", "sqooptest");
+ "sqoop.test.postgresql.username", "sqooptest");
static final String DATABASE_NAME = System.getProperty(
- "sqoop.test.postgresql.connectstring.database", "sqooptest");
- static final String PASSWORD = System.getProperty("sqoop.test.postgresql.connectstring.password");
+ "sqoop.test.postgresql.database", "sqooptest");
+ static final String PASSWORD = System.getProperty("sqoop.test.postgresql.password");
static final String TABLE_NAME = "EMPLOYEES_PG";
static final String NULL_TABLE_NAME = "NULL_EMPLOYEES_PG";
@@ -193,6 +193,8 @@ public class PostgresqlExternalTableImportTest extends ImportJobTestCase {
args.add(CONNECT_STRING);
args.add("--username");
args.add(DATABASE_USER);
+ args.add("--password");
+ args.add(PASSWORD);
args.add("--where");
args.add("id > 1");
args.add("-m");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/33be8445/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java b/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
index 5872d25..ceae47c 100644
--- a/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
+++ b/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
@@ -94,13 +94,13 @@ public class PostgresqlImportTest extends ImportJobTestCase {
"sqoop.test.postgresql.connectstring.host_url",
"jdbc:postgresql://localhost/");
static final String DATABASE_USER = System.getProperty(
- "sqoop.test.postgresql.connectstring.username",
+ "sqoop.test.postgresql.username",
"sqooptest");
static final String DATABASE_NAME = System.getProperty(
- "sqoop.test.postgresql.connectstring.database",
+ "sqoop.test.postgresql.database",
"sqooptest");
static final String PASSWORD = System.getProperty(
- "sqoop.test.postgresql.connectstring.password");
+ "sqoop.test.postgresql.password");
static final String TABLE_NAME = "EMPLOYEES_PG";
static final String NULL_TABLE_NAME = "NULL_EMPLOYEES_PG";
@@ -258,6 +258,8 @@ public class PostgresqlImportTest extends ImportJobTestCase {
args.add(CONNECT_STRING);
args.add("--username");
args.add(DATABASE_USER);
+ args.add("--password");
+ args.add(PASSWORD);
args.add("--where");
args.add("id > 1");
args.add("-m");
@@ -342,6 +344,7 @@ public class PostgresqlImportTest extends ImportJobTestCase {
SqoopOptions options = new SqoopOptions(new Configuration());
options.setConnectString(CONNECT_STRING);
options.setUsername(DATABASE_USER);
+ options.setPassword(PASSWORD);
ConnManager mgr = new PostgresqlManager(options);
String[] tables = mgr.listTables();
[3/3] sqoop git commit: SQOOP-1905: Add --schema option for
import-all-tables and list-tables for DB2 connector
Posted by ma...@apache.org.
SQOOP-1905: Add --schema option for import-all-tables and list-tables for DB2
connector
(Ying Cao via Attila Szabo)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/13ec21cd
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/13ec21cd
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/13ec21cd
Branch: refs/heads/branch-1.4.7
Commit: 13ec21cdaad1804df491155cfcf6382693aa9ac5
Parents: 33be844
Author: Attila Szabo <ma...@apache.org>
Authored: Fri Jul 21 12:09:04 2017 +0200
Committer: Attila Szabo <ma...@apache.org>
Committed: Fri Jul 21 12:19:46 2017 +0200
----------------------------------------------------------------------
.../org/apache/sqoop/manager/Db2Manager.java | 154 ++++++++++-
.../sqoop/manager/GenericJdbcManager.java | 33 +++
.../DB2ImportAllTableWithSchemaManualTest.java | 274 +++++++++++++++++++
3 files changed, 453 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/13ec21cd/src/java/org/apache/sqoop/manager/Db2Manager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/Db2Manager.java b/src/java/org/apache/sqoop/manager/Db2Manager.java
index 52ab05e..61b6868 100644
--- a/src/java/org/apache/sqoop/manager/Db2Manager.java
+++ b/src/java/org/apache/sqoop/manager/Db2Manager.java
@@ -19,12 +19,17 @@ package org.apache.sqoop.manager;
import java.io.IOException;
import java.sql.Connection;
+import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.ParseException;
+import org.apache.sqoop.cli.RelatedOptions;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -54,9 +59,37 @@ public class Db2Manager
private Map<String, String> columnTypeNames;
+ public static final String SCHEMA = "schema";
+
+ /**
+ * Query to list all tables visible to the current user. Note that this list
+ * does not identify the table owners which is required in order to ensure
+ * that the table can be operated on for import/export purposes.
+ */
+
+ public static final String QUERY_LIST_SCHEMA_TABLES = "SELECT DISTINCT NAME FROM SYSIBM.SYSTABLES WHERE CREATOR =? AND TYPE='T' ";
+
+ /**
+ * Query to get the current user's schema for the DB session. Used in case of
+ * wallet logins.
+ */
+ public static final String QUERY_GET_USERSCHEMA =
+ "select current schema from sysibm.sysdummy1";
+
+ /**
+ * DB2 schema that we should use.
+ */
+ private String schema = null;
public Db2Manager(final SqoopOptions opts) {
super(DRIVER_CLASS, opts);
+
+ // Try to parse extra arguments
+ try {
+ this.schema = parseExtraScheArgs(opts.getExtraArgs(),getExtraOptions());
+ } catch (ParseException e) {
+ throw new RuntimeException("Can't parse extra arguments", e);
+ }
}
/**
@@ -107,15 +140,7 @@ public class Db2Manager
// represents schema name.
databases.add(rset.getString(1));
}
- conn.commit();
} catch (SQLException sqle) {
- try {
- if (conn != null) {
- conn.rollback();
- }
- } catch (SQLException ce) {
- LoggingUtils.logAll(LOG, "Failed to rollback transaction", ce);
- }
LoggingUtils.logAll(LOG, "Failed to list databases", sqle);
throw new RuntimeException(sqle);
} finally {
@@ -131,6 +156,105 @@ public class Db2Manager
return databases.toArray(new String[databases.size()]);
}
+ public static String getUserSchema(Connection conn) {
+ Statement stmt = null;
+ ResultSet rset = null;
+ String currSchema = null;
+ try {
+ stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY,
+ ResultSet.CONCUR_READ_ONLY);
+ rset = stmt.executeQuery(QUERY_GET_USERSCHEMA);
+
+ if (rset.next()) {
+ currSchema = rset.getString(1);
+ }
+ } catch (SQLException e) {
+ LoggingUtils.logAll(LOG, "Failed to get user schema", e);
+ } finally {
+ if (rset != null) {
+ try {
+ rset.close();
+ } catch (SQLException ex) {
+ LoggingUtils.logAll(LOG, "Failed to close resultset", ex);
+ }
+ }
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException ex) {
+ LoggingUtils.logAll(LOG, "Failed to close statement", ex);
+ }
+ }
+ }
+ if (currSchema == null) {
+ throw new RuntimeException("Unable to get current user schema");
+ }
+ return currSchema;
+ }
+
+ @Override
+ public String[] listTables() {
+ Connection conn = null;
+ PreparedStatement pStmt = null;
+ ResultSet rset = null;
+ List<String> tables = new ArrayList<String>();
+ String currUserSchema = null;
+
+ try {
+ conn = getConnection();
+ currUserSchema = getUserSchema(conn);
+
+ pStmt = conn.prepareStatement(QUERY_LIST_SCHEMA_TABLES,
+ ResultSet.TYPE_FORWARD_ONLY,
+ ResultSet.CONCUR_READ_ONLY);
+
+ // if user don't provide schema in CLI
+ if (schema == null) {
+ pStmt.setString(1, currUserSchema);
+ } else { //user provide a schema
+ pStmt.setString(1, schema);
+ }
+
+ rset = pStmt.executeQuery();
+
+ if (schema != null && rset == null) {
+ LOG.debug("schema=" + schema
+ + ",maybe not exists in current database");
+ }
+ while (rset.next()) {
+ if(schema == null){
+ tables.add(rset.getString(1));
+ }else{
+ tables.add(schema + "." + rset.getString(1));
+ }
+ }
+ } catch (SQLException e) {
+ LoggingUtils.logAll(LOG, "Failed to list tables", e);
+ } finally {
+ if (rset != null) {
+ try {
+ rset.close();
+ } catch (SQLException ex) {
+ LoggingUtils.logAll(LOG, "Failed to close resultset", ex);
+ }
+ }
+ if (pStmt != null) {
+ try {
+ pStmt.close();
+ } catch (SQLException ex) {
+ LoggingUtils.logAll(LOG, "Failed to close statement", ex);
+ }
+ }
+
+ try {
+ close();
+ } catch (SQLException ex) {
+ LoggingUtils.logAll(LOG, "Unable to discard connection", ex);
+ }
+ }
+ return tables.toArray(new String[tables.size()]);
+ }
+
/**
* Return hive type for SQL type.
*
@@ -224,4 +348,18 @@ public class Db2Manager
return null;
}
+ /**
+ * Create related options for PostgreSQL extra parameters.
+ * @return
+ */
+ @SuppressWarnings("static-access")
+ protected RelatedOptions getExtraOptions() {
+ // Connection args (common)
+ RelatedOptions extraOptions = new RelatedOptions("DB2 extra options:");
+ extraOptions.addOption(OptionBuilder.withArgName("string").hasArg()
+ .withDescription("Optional schema name").withLongOpt(SCHEMA)
+ .create("schema"));
+ return extraOptions;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/13ec21cd/src/java/org/apache/sqoop/manager/GenericJdbcManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/GenericJdbcManager.java b/src/java/org/apache/sqoop/manager/GenericJdbcManager.java
index 2113a5f..f38bcc5 100644
--- a/src/java/org/apache/sqoop/manager/GenericJdbcManager.java
+++ b/src/java/org/apache/sqoop/manager/GenericJdbcManager.java
@@ -21,10 +21,16 @@ package org.apache.sqoop.manager;
import java.sql.Connection;
import java.sql.SQLException;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.cloudera.sqoop.SqoopOptions;
+import org.apache.sqoop.cli.RelatedOptions;
/**
* Database manager that is connects to a generic JDBC-compliant
@@ -39,6 +45,7 @@ public class GenericJdbcManager
private String jdbcDriverClass;
private Connection connection;
+ private static final String SCHEMA = "schema";
public GenericJdbcManager(final String driverClass, final SqoopOptions opts) {
super(opts);
@@ -84,5 +91,31 @@ public class GenericJdbcManager
public String getDriverClass() {
return jdbcDriverClass;
}
+
+ public String parseExtraScheArgs(String[] args,RelatedOptions opts) throws ParseException {
+ // No-op when no extra arguments are present
+ if (args == null || args.length == 0) {
+ return null;
+ }
+
+ // We do not need extended abilities of SqoopParser, so we're using
+ // Gnu parser instead.
+ CommandLineParser parser = new GnuParser();
+ CommandLine cmdLine = parser.parse(opts, args, true);
+
+ //Apply parsed arguments
+ return applyExtraScheArguments(cmdLine);
+ }
+
+ public String applyExtraScheArguments(CommandLine cmdLine) {
+ if (cmdLine.hasOption(SCHEMA)) {
+ String schemaName = cmdLine.getOptionValue(SCHEMA);
+ LOG.info("We will use schema " + schemaName);
+
+ return schemaName;
+ }
+
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/13ec21cd/src/test/org/apache/sqoop/manager/db2/DB2ImportAllTableWithSchemaManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/db2/DB2ImportAllTableWithSchemaManualTest.java b/src/test/org/apache/sqoop/manager/db2/DB2ImportAllTableWithSchemaManualTest.java
new file mode 100644
index 0000000..db6e6bf
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/db2/DB2ImportAllTableWithSchemaManualTest.java
@@ -0,0 +1,274 @@
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.manager.db2;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.manager.Db2Manager;
+import org.apache.sqoop.tool.ImportAllTablesTool;
+import org.apache.sqoop.Sqoop;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.testutil.CommonArgs;
+import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import com.cloudera.sqoop.tool.SqoopTool;
+import com.cloudera.sqoop.util.FileListing;
+import org.apache.sqoop.util.LoggingUtils;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test the DB2 XML data type.
+ *
+ * This uses JDBC to import data from an DB2 database into HDFS.
+ *
+ * Since this requires an DB2 Server installation,
+ * this class is named in such a way that Sqoop's default QA process does
+ * not run it. You need to run this manually with
+ * -Dtestcase=DB2ImportAllTableWithSchema
+
+ * You need to put DB2 JDBC driver library (db2jcc.jar) in a location
+ * where Sqoop will be able to access it (since this library cannot be checked
+ * into Apache's tree for licensing reasons).
+ *
+ * To set up your test environment:
+ * Install DB2 Express 9.7 C server.
+ * Create a database SQOOP
+ * Create a login SQOOP with password PASSWORD and grant all
+ * access for database SQOOP to user SQOOP.
+ */
+public class DB2ImportAllTableWithSchemaManualTest extends ImportJobTestCase {
+
+ public static final Log LOG = LogFactory.getLog(
+ DB2ImportAllTableWithSchemaManualTest.class.getName());
+
+ static final String HOST_URL = System.getProperty(
+ "sqoop.test.db2.connectstring.host_url",
+ "jdbc:db2://9.30.245.234:60000");
+
+ static final String DATABASE_NAME = System.getProperty(
+ "sqoop.test.db2.connectstring.database",
+ "TESTDB");
+ static final String DATABASE_USER = System.getProperty(
+ "sqoop.test.db2.connectstring.username",
+ "DB2FENC1");
+ static final String DATABASE_PASSWORD = System.getProperty(
+ "sqoop.test.db2.connectstring.password",
+ "DB2FENC1");
+
+ static final String TABLE_NAME = "TEST.COMPANY";
+ static final String TABLE_SCHEMA = "TEST";
+ static final String CONNECT_STRING = HOST_URL
+ + "/" + DATABASE_NAME;
+ static String ExpectedResults =
+ "1,doc1";
+
+
+ String [] extraArgs = { "--",
+ "--schema", TABLE_SCHEMA,
+ };
+
+ static {
+ LOG.info("Using DB2 CONNECT_STRING HOST_URL is : "+HOST_URL);
+ LOG.info("Using DB2 CONNECT_STRING: " + CONNECT_STRING);
+ }
+
+ // instance variables populated during setUp, used during tests
+ private Db2Manager manager;
+
+ protected String getTableName() {
+ return TABLE_NAME;
+ }
+
+
+ @Before
+ public void setUp() {
+ super.setUp();
+
+ SqoopOptions options = new SqoopOptions(CONNECT_STRING, getTableName());
+ options.setUsername(DATABASE_USER);
+ options.setPassword(DATABASE_PASSWORD);
+
+ manager = new Db2Manager(options);
+
+ // Drop the existing table, if there is one.
+ Connection conn = null;
+ Statement stmt = null;
+ try {
+ conn = manager.getConnection();
+ stmt = conn.createStatement();
+ stmt.execute("DROP TABLE " + getTableName());
+ } catch (SQLException sqlE) {
+ LoggingUtils.logAll(LOG, "Table was not dropped: ", sqlE);
+ } finally {
+ try {
+ if (null != stmt) {
+ stmt.close();
+ }
+ } catch (Exception ex) {
+ LOG.warn("Exception while closing stmt", ex);
+ }
+ }
+
+ // Create and populate table
+ try {
+ conn = manager.getConnection();
+ conn.setAutoCommit(false);
+ stmt = conn.createStatement();
+
+ // create the database table and populate it with data.
+ stmt.executeUpdate("CREATE TABLE " + getTableName() + " ("
+ + "ID int, "
+ + "DOCNAME VARCHAR(20))");
+
+ stmt.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
+ + "1,'doc1' "
+ + " )");
+ conn.commit();
+ } catch (SQLException sqlE) {
+ LoggingUtils.logAll(LOG, "Encountered SQL Exception: ", sqlE);
+ } finally {
+ try {
+ if (null != stmt) {
+ stmt.close();
+ }
+ } catch (Exception ex) {
+ LOG.warn("Exception while closing connection/stmt", ex);
+ }
+ }
+ }
+
+ @After
+ public void tearDown() {
+ super.tearDown();
+ try {
+ manager.close();
+ } catch (SQLException sqlE) {
+ LoggingUtils.logAll(LOG, "Got SQLException: ", sqlE);
+ }
+ }
+
+ @Test
+ public void testDb2Import() throws IOException {
+
+ runDb2Test(ExpectedResults);
+
+ }
+
+ private String [] getArgv() {
+ ArrayList<String> args = new ArrayList<String>();
+
+ CommonArgs.addHadoopFlags(args);
+ args.add("--connect");
+ args.add(CONNECT_STRING);
+ args.add("--username");
+ args.add(DATABASE_USER);
+ args.add("--password");
+ args.add(DATABASE_PASSWORD);
+ args.add("--warehouse-dir");
+ args.add(getWarehouseDir());
+
+ args.add("--m");
+ args.add("1");
+
+ for (String arg : extraArgs) {
+ args.add(arg);
+ }
+
+ return args.toArray(new String[0]);
+ }
+
+ private void runDb2Test(String expectedResults) throws IOException {
+
+ Path warehousePath = new Path(this.getWarehouseDir());
+ Path tablePath = new Path(warehousePath, getTableName());
+ Path filePath = new Path(tablePath, "part-m-00000");
+
+ File tableFile = new File(getTableName().toString());
+ if (tableFile.exists() && tableFile.isDirectory()) {
+ // remove the directory before running the import.
+ FileListing.recursiveDeleteDir(tableFile);
+ }
+
+ String [] argv = getArgv();
+ try {
+ runImportAll(argv);
+ } catch (IOException ioe) {
+ LOG.error("Got IOException during import: " + ioe.getMessage());
+ }
+
+ File f = new File(filePath.toString());
+ assertTrue("Could not find imported data file", f.exists());
+ BufferedReader r = null;
+ try {
+ // Read through the file and make sure it's all there.
+ r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
+ assertEquals(expectedResults, r.readLine());
+ } catch (IOException ioe) {
+ LOG.error("Got IOException verifying results: " + ioe.getMessage());
+ } finally {
+ IOUtils.closeStream(r);
+ }
+ }
+
+ private void runImportAll(SqoopTool tool,String [] argv) throws IOException {
+ // run the tool through the normal entry-point.
+ int ret;
+ try {
+ Configuration conf = getConf();
+ SqoopOptions opts = getSqoopOptions(conf);
+ Sqoop sqoop = new Sqoop(tool, conf, opts);
+ ret = Sqoop.runSqoop(sqoop, argv);
+ //ret = tool.run(opts);
+ } catch (Exception e) {
+ LOG.error("Got exception running Sqoop: " + e.toString());
+ ret = 1;
+ }
+
+ // expect a successful return.
+ if (0 != ret) {
+ throw new IOException("Failure during job; return status " + ret);
+ }
+ }
+
+ /** run an import using the default ImportTool. */
+ protected void runImportAll(String [] argv) throws IOException {
+ runImportAll(new ImportAllTablesTool(), argv);
+ }
+}