You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by bl...@apache.org on 2011/11/01 22:01:11 UTC
svn commit: r1196272 [1/4] - in /incubator/sqoop/trunk/src/java:
com/cloudera/sqoop/manager/ org/apache/sqoop/manager/
Author: blee
Date: Tue Nov 1 21:01:09 2011
New Revision: 1196272
URL: http://svn.apache.org/viewvc?rev=1196272&view=rev
Log:
SQOOP-378: Migrate manager package to new name space
Added:
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/CatalogQueryManager.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/Db2Manager.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/DefaultManagerFactory.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/DirectMySQLManager.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/ExportJobContext.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/GenericJdbcManager.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/HsqldbManager.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/ImportJobContext.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/InformationSchemaManager.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/ManagerFactory.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLUtils.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/PostgresqlManager.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/SQLServerManager.java (with props)
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/SqlManager.java (with props)
Modified:
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/CatalogQueryManager.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ConnManager.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/Db2Manager.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/DefaultManagerFactory.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/DirectMySQLManager.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/DirectPostgresqlManager.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ExportJobContext.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/GenericJdbcManager.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/HsqldbManager.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ImportJobContext.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/InformationSchemaManager.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ManagerFactory.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/MySQLManager.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/MySQLUtils.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/OracleManager.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/PostgresqlManager.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SQLServerManager.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/CatalogQueryManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/CatalogQueryManager.java?rev=1196272&r1=1196271&r2=1196272&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/CatalogQueryManager.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/CatalogQueryManager.java Tue Nov 1 21:01:09 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2011 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,222 +18,17 @@
package com.cloudera.sqoop.manager;
-import java.sql.Connection;
-import java.sql.Statement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import com.cloudera.sqoop.SqoopOptions;
/**
- * Database manager that queries catalog tables directly
- * (instead of metadata calls) to retrieve information.
+ * @deprecated Moving to use org.apache.sqoop namespace.
*/
-public abstract class CatalogQueryManager extends GenericJdbcManager {
-
- public static final Log LOG = LogFactory.getLog(
- CatalogQueryManager.class.getName());
+public abstract class CatalogQueryManager
+ extends org.apache.sqoop.manager.CatalogQueryManager {
public CatalogQueryManager(final String driverClass,
final SqoopOptions opts) {
super(driverClass, opts);
}
-
- protected abstract String getListDatabasesQuery();
- @Override
- public String[] listDatabases() {
- Connection c = null;
- Statement s = null;
- ResultSet rs = null;
- List<String> databases = new ArrayList<String>();
- try {
- c = getConnection();
- s = c.createStatement();
- rs = s.executeQuery(getListDatabasesQuery());
- while (rs.next()) {
- databases.add(rs.getString(1));
- }
- c.commit();
- } catch (SQLException sqle) {
- try {
- if (c != null) {
- c.rollback();
- }
- } catch (SQLException ce) {
- LOG.error("Failed to rollback transaction", ce);
- }
- LOG.error("Failed to list databases", sqle);
- throw new RuntimeException(sqle);
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (SQLException re) {
- LOG.error("Failed to close resultset", re);
- }
- }
- if (s != null) {
- try {
- s.close();
- } catch (SQLException se) {
- LOG.error("Failed to close statement", se);
- }
- }
- }
-
- return databases.toArray(new String[databases.size()]);
- }
-
- protected abstract String getListTablesQuery();
- @Override
- public String[] listTables() {
- Connection c = null;
- Statement s = null;
- ResultSet rs = null;
- List<String> tables = new ArrayList<String>();
- try {
- c = getConnection();
- s = c.createStatement();
- rs = s.executeQuery(getListTablesQuery());
- while (rs.next()) {
- tables.add(rs.getString(1));
- }
- c.commit();
- } catch (SQLException sqle) {
- try {
- if (c != null) {
- c.rollback();
- }
- } catch (SQLException ce) {
- LOG.error("Failed to rollback transaction", ce);
- }
- LOG.error("Failed to list tables", sqle);
- throw new RuntimeException(sqle);
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (SQLException re) {
- LOG.error("Failed to close resultset", re);
- }
- }
- if (s != null) {
- try {
- s.close();
- } catch (SQLException se) {
- LOG.error("Failed to close statement", se);
- }
- }
- }
-
- return tables.toArray(new String[tables.size()]);
- }
-
- protected abstract String getListColumnsQuery(String tableName);
- @Override
- public String[] getColumnNames(String tableName) {
- Connection c = null;
- Statement s = null;
- ResultSet rs = null;
- List<String> columns = new ArrayList<String>();
- try {
- c = getConnection();
- s = c.createStatement();
- rs = s.executeQuery(getListColumnsQuery(tableName));
- while (rs.next()) {
- columns.add(rs.getString(1));
- }
- c.commit();
- } catch (SQLException sqle) {
- try {
- if (c != null) {
- c.rollback();
- }
- } catch (SQLException ce) {
- LOG.error("Failed to rollback transaction", ce);
- }
- LOG.error("Failed to list columns", sqle);
- throw new RuntimeException(sqle);
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (SQLException re) {
- LOG.error("Failed to close resultset", re);
- }
- }
- if (s != null) {
- try {
- s.close();
- } catch (SQLException se) {
- LOG.error("Failed to close statement", se);
- }
- }
- }
-
- return columns.toArray(new String[columns.size()]);
- }
-
- protected abstract String getPrimaryKeyQuery(String tableName);
- @Override
- public String getPrimaryKey(String tableName) {
- Connection c = null;
- Statement s = null;
- ResultSet rs = null;
- List<String> columns = new ArrayList<String>();
- try {
- c = getConnection();
- s = c.createStatement();
- rs = s.executeQuery(getPrimaryKeyQuery(tableName));
- while (rs.next()) {
- columns.add(rs.getString(1));
- }
- c.commit();
- } catch (SQLException sqle) {
- try {
- if (c != null) {
- c.rollback();
- }
- } catch (SQLException ce) {
- LOG.error("Failed to rollback transaction", ce);
- }
- LOG.error("Failed to list primary key", sqle);
- throw new RuntimeException(sqle);
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (SQLException re) {
- LOG.error("Failed to close resultset", re);
- }
- }
- if (s != null) {
- try {
- s.close();
- } catch (SQLException se) {
- LOG.error("Failed to close statement", se);
- }
- }
- }
-
- if (columns.size() == 0) {
- // Table has no primary key
- return null;
- }
-
- if (columns.size() > 1) {
- // The primary key is multi-column primary key. Warn the user.
- LOG.warn("The table " + tableName + " "
- + "contains a multi-column primary key. Sqoop will default to "
- + "the column " + columns.get(0) + " only for this job.");
- }
-
- return columns.get(0);
- }
}
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ConnManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ConnManager.java?rev=1196272&r1=1196271&r2=1196272&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ConnManager.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ConnManager.java Tue Nov 1 21:01:09 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2011 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,367 +18,9 @@
package com.cloudera.sqoop.manager;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Timestamp;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.StringTokenizer;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.util.ExportException;
-import com.cloudera.sqoop.util.ImportException;
-
/**
- * Abstract interface that manages connections to a database.
- * The implementations of this class drive the actual discussion with
- * the database about table formats, etc.
+ * @deprecated Moving to use org.apache.sqoop namespace.
*/
-public abstract class ConnManager {
-
- public static final Log LOG = LogFactory.getLog(SqlManager.class.getName());
-
- /**
- * Return a list of all databases on a server.
- */
- public abstract String [] listDatabases();
-
- /**
- * Return a list of all tables in a database.
- */
- public abstract String [] listTables();
-
- /**
- * Return a list of column names in a table in the order returned by the db.
- */
- public abstract String [] getColumnNames(String tableName);
-
- /**
- * Return a list of column names in query in the order returned by the db.
- */
- public String [] getColumnNamesForQuery(String query) {
- LOG.error("This database does not support free-form query column names.");
- return null;
- }
-
- /**
- * Return the name of the primary key for a table, or null if there is none.
- */
- public abstract String getPrimaryKey(String tableName);
-
- /**
- * Return java type for SQL type.
- * @param sqlType sql type
- * @return java type
- */
- public abstract String toJavaType(int sqlType);
-
- /**
- * Return hive type for SQL type.
- * @param sqlType sql type
- * @return hive type
- */
- public abstract String toHiveType(int sqlType);
-
- /**
- * Return an unordered mapping from colname to sqltype for
- * all columns in a table.
- *
- * The Integer type id is a constant from java.sql.Types
- */
- public abstract Map<String, Integer> getColumnTypes(String tableName);
-
- /**
- * Return an unordered mapping from colname to sqltype for
- * all columns in a table or query.
- *
- * The Integer type id is a constant from java.sql.Types
- *
- * @param tableName the name of the table
- * @param sqlQuery the SQL query to use if tableName is null
- */
- public Map<String, Integer> getColumnTypes(String tableName,
- String sqlQuery) throws IOException {
- Map<String, Integer> columnTypes;
- if (null != tableName) {
- // We're generating a class based on a table import.
- columnTypes = getColumnTypes(tableName);
- } else {
- // This is based on an arbitrary query.
- String query = sqlQuery;
- if (query.indexOf(SqlManager.SUBSTITUTE_TOKEN) == -1) {
- throw new IOException("Query [" + query + "] must contain '"
- + SqlManager.SUBSTITUTE_TOKEN + "' in WHERE clause.");
- }
-
- columnTypes = getColumnTypesForQuery(query);
- }
- return columnTypes;
- }
-
- /**
- * This method allows various connection managers to indicate if they support
- * staging data for export jobs. The managers that do support this must
- * override this method and return <tt>true</tt>.
- *
- * @return true if the connection manager supports staging data for export
- * use-case.
- */
- public boolean supportsStagingForExport() {
- return false;
- }
-
- /**
- * Returns the count of all rows that exist in the given table.
- * @param tableName the name of the table which will be queried.
- * @return the number of rows present in the given table.
- * @throws SQLException if an error occurs during execution
- * @throws UnsupportedOperationException if the connection manager does not
- * support this operation.
- */
- public long getTableRowCount(String tableName) throws SQLException {
- throw new UnsupportedOperationException();
- }
-
- /**
- * Deletes all records from the given table. This method is invoked during
- * and export run when a staging table is specified. The staging table is
- * cleaned before the commencement of export job, and after the data has
- * been moved to the target table.
- * @param tableName name of the table which will be emptied.
- * @throws SQLException if an error occurs during execution
- * @throws UnsupportedOperationException if the connection manager does not
- * support this operation.
- */
- public void deleteAllRecords(String tableName) throws SQLException {
- throw new UnsupportedOperationException();
- }
-
- /**
- * Migrates all records from the given <tt>fromTable</tt> to the target
- * <tt>toTable</tt>. This method is invoked as a last step of an export
- * run where the staging is used to collect data before pushing it into the
- * target table.
- * @param fromTable the name of the staging table
- * @param toTable the name of the target table
- * @throws SQLException if an error occurs during execution
- * @throws UnsupportedOperationException if the connection manager does not
- * support this operation.
- */
- public void migrateData(String fromTable, String toTable)
- throws SQLException {
- throw new UnsupportedOperationException();
- }
-
- /**
- * Return an unordered mapping from colname to sqltype for
- * all columns in a query.
- *
- * The Integer type id is a constant from java.sql.Types
- */
- public Map<String, Integer> getColumnTypesForQuery(String query) {
- LOG.error("This database does not support free-form query column types.");
- return null;
- }
-
- /**
- * Execute a SQL statement to read the named set of columns from a table.
- * If columns is null, all columns from the table are read. This is a direct
- * (non-parallelized) read of the table back to the current client.
- * The client is responsible for calling ResultSet.close() when done with the
- * returned ResultSet object, and for calling release() after that to free
- * internal state.
- */
- public abstract ResultSet readTable(String tableName, String [] columns)
- throws SQLException;
-
- /**
- * @return the actual database connection.
- */
- public abstract Connection getConnection() throws SQLException;
-
- /**
- * @return a string identifying the driver class to load for this
- * JDBC connection type.
- */
- public abstract String getDriverClass();
-
- /**
- * Execute a SQL statement 's' and print its results to stdout.
- */
- public abstract void execAndPrint(String s);
-
- /**
- * Perform an import of a table from the database into HDFS.
- */
- public abstract void importTable(ImportJobContext context)
- throws IOException, ImportException;
-
- /**
- * Perform an import of a free-form query from the database into HDFS.
- */
- public void importQuery(ImportJobContext context)
- throws IOException, ImportException {
- throw new ImportException(
- "This database only supports table-based imports.");
- }
-
- /**
- * When using a column name in a generated SQL query, how (if at all)
- * should we escape that column name? e.g., a column named "table"
- * may need to be quoted with backtiks: "`table`".
- *
- * @param colName the column name as provided by the user, etc.
- * @return how the column name should be rendered in the sql text.
- */
- public String escapeColName(String colName) {
- return colName;
- }
-
- /**
- * When using a table name in a generated SQL query, how (if at all)
- * should we escape that column name? e.g., a table named "table"
- * may need to be quoted with backtiks: "`table`".
- *
- * @param tableName the table name as provided by the user, etc.
- * @return how the table name should be rendered in the sql text.
- */
- public String escapeTableName(String tableName) {
- return tableName;
- }
-
- /**
- * Perform any shutdown operations on the connection.
- */
- public abstract void close() throws SQLException;
-
- /**
- * Export data stored in HDFS into a table in a database.
- * This inserts new rows into the target table.
- */
- public void exportTable(ExportJobContext context)
- throws IOException, ExportException {
- throw new ExportException("This database does not support exports");
- }
-
- /**
- * Export updated data stored in HDFS into a database table.
- * This updates existing rows in the target table, based on the
- * updateKeyCol specified in the context's SqoopOptions.
- */
- public void updateTable(ExportJobContext context)
- throws IOException, ExportException {
- throw new ExportException("This database does not support updates");
- }
-
- /**
- * Export data stored in HDFS into a table in a database.
- * This may update or insert rows into the target table depending on
- * whether rows already exist in the target table or not.
- */
- public void upsertTable(ExportJobContext context)
- throws IOException, ExportException {
- throw new ExportException("Mixed update/insert is not supported"
- + " against the target database yet");
- }
-
- /**
- * Configure database output column ordering explicitly for code generator.
- * The code generator should generate the DBWritable.write(PreparedStatement)
- * method with columns exporting in this order.
- */
- public void configureDbOutputColumns(SqoopOptions options) {
- // We're in update mode. We need to explicitly set the database output
- // column ordering in the codeGenerator. The UpdateKeyCol must come
- // last, because the UPDATE-based OutputFormat will generate the SET
- // clause followed by the WHERE clause, and the SqoopRecord needs to
- // serialize to this layout.
- Set<String> updateKeys = new LinkedHashSet<String>();
- Set<String> updateKeysUppercase = new HashSet<String>();
- String updateKeyValue = options.getUpdateKeyCol();
- StringTokenizer stok = new StringTokenizer(updateKeyValue, ",");
- while (stok.hasMoreTokens()) {
- String nextUpdateColumn = stok.nextToken().trim();
- if (nextUpdateColumn.length() > 0) {
- updateKeys.add(nextUpdateColumn);
- updateKeysUppercase.add(nextUpdateColumn.toUpperCase());
- } else {
- throw new RuntimeException("Invalid update key column value specified"
- + ": '" + updateKeyValue + "'");
- }
- }
- String [] allColNames = getColumnNames(options.getTableName());
- List<String> dbOutCols = new ArrayList<String>();
- for (String col : allColNames) {
- if (!updateKeysUppercase.contains(col.toUpperCase())) {
- dbOutCols.add(col); // add non-key columns to the output order list.
- }
- }
-
- // Then add the update key column last.
- dbOutCols.addAll(updateKeys);
- options.setDbOutputColumns(dbOutCols.toArray(
- new String[dbOutCols.size()]));
- }
-
- /**
- * If a method of this ConnManager has returned a ResultSet to you,
- * you are responsible for calling release() after you close the
- * ResultSet object, to free internal resources. ConnManager
- * implementations do not guarantee the ability to have multiple
- * returned ResultSets available concurrently. Requesting a new
- * ResultSet from a ConnManager may cause other open ResulSets
- * to close.
- */
- public abstract void release();
-
- /**
- * Return the current time from the perspective of the database server.
- * Return null if this cannot be accessed.
- */
- public Timestamp getCurrentDbTimestamp() {
- LOG.warn("getCurrentDbTimestamp(): Using local system timestamp.");
- return new Timestamp(System.currentTimeMillis());
- }
-
- /**
- * Given a non-null Timestamp, return the quoted string that can
- * be inserted into a SQL statement, representing that timestamp.
- */
- public String timestampToQueryString(Timestamp ts) {
- return "'" + ts + "'";
- }
-
- /**
- * Given a date/time, return the quoted string that can
- * be inserted into a SQL statement, representing that date/time.
- */
- public String datetimeToQueryString(String datetime, int columnType) {
- return "'" + datetime + "'";
- }
-
- /**
- * This method allows the ConnManager to override the creation of an
- * input-bounds query that is used to create splits when running import
- * based on free-form query. Any non-null return value is used, whereas a null
- * return value indicates that the default input bounds query should be
- * used.
- * @param splitByCol the column name to split on.
- * @param sanitizedQuery the sanitized input query specified by user.
- * @return an input-bounds query or <tt>null</tt> if default query is
- * acceptable.
- */
- public String getInputBoundsQuery(String splitByCol, String sanitizedQuery) {
- return null;
- }
+public abstract class ConnManager
+ extends org.apache.sqoop.manager.ConnManager {
}
-
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/Db2Manager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/Db2Manager.java?rev=1196272&r1=1196271&r2=1196272&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/Db2Manager.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/Db2Manager.java Tue Nov 1 21:01:09 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2011 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -19,93 +17,16 @@
*/
package com.cloudera.sqoop.manager;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.mapreduce.ExportBatchOutputFormat;
-import com.cloudera.sqoop.mapreduce.JdbcExportJob;
-import com.cloudera.sqoop.util.ExportException;
/**
- * Manages connections to DB2 databases. Requires the DB2 JDBC driver.
+ * @deprecated Moving to use org.apache.sqoop namespace.
*/
-public class Db2Manager extends GenericJdbcManager {
-
- public static final Log LOG = LogFactory.getLog(
- Db2Manager.class.getName());
-
- // driver class to ensure is loaded when making db connection.
- private static final String DRIVER_CLASS =
- "com.ibm.db2.jcc.DB2Driver";
+public class Db2Manager
+ extends org.apache.sqoop.manager.Db2Manager {
public Db2Manager(final SqoopOptions opts) {
- super(DRIVER_CLASS, opts);
+ super(opts);
}
- /**
- * Export data stored in HDFS into a table in a database.
- */
- @Override
- public void exportTable(ExportJobContext context)
- throws IOException, ExportException {
- context.setConnManager(this);
- JdbcExportJob exportJob = new JdbcExportJob(context, null, null,
- ExportBatchOutputFormat.class);
- exportJob.runExport();
- }
-
- /**
- * DB2 does not support the CURRENT_TIMESTAMP() function. Instead
- * it uses the sysibm schema for timestamp lookup.
- */
- @Override
- public String getCurTimestampQuery() {
- return "SELECT CURRENT TIMESTAMP FROM SYSIBM.SYSDUMMY1 WITH UR";
- }
-
- @Override
- public String[] listDatabases() {
- Connection conn = null;
- ResultSet rset = null;
- List<String> databases = new ArrayList<String>();
- try {
- conn = getConnection();
- rset = conn.getMetaData().getSchemas();
- while (rset.next()) {
- // The ResultSet contains two columns - TABLE_SCHEM(1),
- // TABLE_CATALOG(2). We are only interested in TABLE_SCHEM which
- // represents schema name.
- databases.add(rset.getString(1));
- }
- conn.commit();
- } catch (SQLException sqle) {
- try {
- if (conn != null) {
- conn.rollback();
- }
- } catch (SQLException ce) {
- LOG.error("Failed to rollback transaction", ce);
- }
- LOG.error("Failed to list databases", sqle);
- throw new RuntimeException(sqle);
- } finally {
- if (rset != null) {
- try {
- rset.close();
- } catch (SQLException re) {
- LOG.error("Failed to close resultset", re);
- }
- }
- }
-
- return databases.toArray(new String[databases.size()]);
- }
}
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/DefaultManagerFactory.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/DefaultManagerFactory.java?rev=1196272&r1=1196271&r2=1196272&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/DefaultManagerFactory.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/DefaultManagerFactory.java Tue Nov 1 21:01:09 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2011 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,112 +18,10 @@
package com.cloudera.sqoop.manager;
-import java.lang.reflect.Constructor;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.metastore.JobData;
-
/**
- * Contains instantiation code for all ConnManager implementations
- * shipped and enabled by default in Sqoop.
+ * @deprecated Moving to use org.apache.sqoop namespace.
*/
-public final class DefaultManagerFactory extends ManagerFactory {
-
- public static final Log LOG = LogFactory.getLog(
- DefaultManagerFactory.class.getName());
-
- @Override
- public ConnManager accept(JobData data) {
- SqoopOptions options = data.getSqoopOptions();
- String manualDriver = options.getDriverClassName();
- if (manualDriver != null) {
- // User has manually specified JDBC implementation with --driver.
- // Just use GenericJdbcManager.
- return new GenericJdbcManager(manualDriver, options);
- }
-
- if (null != options.getConnManagerClassName()){
- String className = options.getConnManagerClassName();
- ConnManager connManager = null;
- try {
- Class<ConnManager> cls = (Class<ConnManager>) Class.forName(className);
- Constructor<ConnManager> constructor =
- cls.getDeclaredConstructor(SqoopOptions.class);
- connManager = constructor.newInstance(options);
- } catch (Exception e) {
- System.err
- .println("problem finding the connection manager for class name :"
- + className);
- // Log the stack trace for this exception
- LOG.debug(e.getMessage(), e);
- // Print exception message.
- System.err.println(e.getMessage());
- }
- return connManager;
- }
-
- String connectStr = options.getConnectString();
-
- // java.net.URL follows RFC-2396 literally, which does not allow a ':'
- // character in the scheme component (section 3.1). JDBC connect strings,
- // however, commonly have a multi-scheme addressing system. e.g.,
- // jdbc:mysql://...; so we cannot parse the scheme component via URL
- // objects. Instead, attempt to pull out the scheme as best as we can.
-
- // First, see if this is of the form [scheme://hostname-and-etc..]
- int schemeStopIdx = connectStr.indexOf("//");
- if (-1 == schemeStopIdx) {
- // If no hostname start marker ("//"), then look for the right-most ':'
- // character.
- schemeStopIdx = connectStr.lastIndexOf(':');
- if (-1 == schemeStopIdx) {
- // Warn that this is nonstandard. But we should be as permissive
- // as possible here and let the ConnectionManagers themselves throw
- // out the connect string if it doesn't make sense to them.
- LOG.warn("Could not determine scheme component of connect string");
-
- // Use the whole string.
- schemeStopIdx = connectStr.length();
- }
- }
-
- String scheme = connectStr.substring(0, schemeStopIdx);
-
- if (null == scheme) {
- // We don't know if this is a mysql://, hsql://, etc.
- // Can't do anything with this.
- LOG.warn("Null scheme associated with connect string.");
- return null;
- }
-
- LOG.debug("Trying with scheme: " + scheme);
-
- if (scheme.equals("jdbc:mysql:")) {
- if (options.isDirect()) {
- return new DirectMySQLManager(options);
- } else {
- return new MySQLManager(options);
- }
- } else if (scheme.equals("jdbc:postgresql:")) {
- if (options.isDirect()) {
- return new DirectPostgresqlManager(options);
- } else {
- return new PostgresqlManager(options);
- }
- } else if (scheme.startsWith("jdbc:hsqldb:")) {
- return new HsqldbManager(options);
- } else if (scheme.startsWith("jdbc:oracle:")) {
- return new OracleManager(options);
- } else if (scheme.startsWith("jdbc:sqlserver:")) {
- return new SQLServerManager(options);
- } else if (scheme.startsWith("jdbc:db2:")) {
- return new Db2Manager(options);
- } else {
- return null;
- }
- }
+public final class DefaultManagerFactory
+ extends org.apache.sqoop.manager.DefaultManagerFactory {
}
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/DirectMySQLManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/DirectMySQLManager.java?rev=1196272&r1=1196271&r2=1196272&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/DirectMySQLManager.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/DirectMySQLManager.java Tue Nov 1 21:01:09 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2011 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,94 +18,17 @@
package com.cloudera.sqoop.manager;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.mapreduce.MySQLDumpImportJob;
-import com.cloudera.sqoop.mapreduce.MySQLExportJob;
-import com.cloudera.sqoop.util.ImportException;
-import com.cloudera.sqoop.util.ExportException;
/**
- * Manages direct connections to MySQL databases
- * so we can use mysqldump to get really fast dumps.
+ * @deprecated Moving to use org.apache.sqoop namespace.
*/
-public class DirectMySQLManager extends MySQLManager {
-
- public static final Log LOG = LogFactory.getLog(
- DirectMySQLManager.class.getName());
+public class DirectMySQLManager
+ extends org.apache.sqoop.manager.DirectMySQLManager {
public DirectMySQLManager(final SqoopOptions options) {
super(options);
}
- /**
- * Import the table into HDFS by using mysqldump to pull out the data from
- * the database and upload the files directly to HDFS.
- */
- @Override
- public void importTable(ImportJobContext context)
- throws IOException, ImportException {
-
- context.setConnManager(this);
- if (context.getOptions().getColumns() != null) {
- LOG.warn("Direct-mode import from MySQL does not support column");
- LOG.warn("selection. Falling back to JDBC-based import.");
- // Don't warn them "This could go faster..."
- MySQLManager.markWarningPrinted();
- // Use JDBC-based importTable() method.
- super.importTable(context);
- return;
- }
-
- String tableName = context.getTableName();
- String jarFile = context.getJarFile();
- SqoopOptions options = context.getOptions();
-
- MySQLDumpImportJob importer = null;
- try {
- importer = new MySQLDumpImportJob(options, context);
- } catch (ClassNotFoundException cnfe) {
- throw new IOException("Could not load required classes", cnfe);
- }
-
- String splitCol = getSplitColumn(options, tableName);
- if (null == splitCol && options.getNumMappers() > 1) {
- // Can't infer a primary key.
- throw new ImportException("No primary key could be found for table "
- + tableName + ". Please specify one with --split-by or perform "
- + "a sequential import with '-m 1'.");
- }
-
- LOG.info("Beginning mysqldump fast path import");
-
- if (options.getFileLayout() != SqoopOptions.FileLayout.TextFile) {
- // TODO(aaron): Support SequenceFile-based load-in.
- LOG.warn("File import layout " + options.getFileLayout()
- + " is not supported by");
- LOG.warn("MySQL direct import; import will proceed as text files.");
- }
-
- importer.runImport(tableName, jarFile, splitCol, options.getConf());
- }
-
- /**
- * Export the table from HDFS by using mysqlimport to insert the data
- * back into the database.
- */
- @Override
- public void exportTable(ExportJobContext context)
- throws IOException, ExportException {
- context.setConnManager(this);
- MySQLExportJob exportJob = new MySQLExportJob(context);
- exportJob.runExport();
- }
-
- @Override
- public boolean supportsStagingForExport() {
- return false;
- }
}
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/DirectPostgresqlManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/DirectPostgresqlManager.java?rev=1196272&r1=1196271&r2=1196272&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/DirectPostgresqlManager.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/DirectPostgresqlManager.java Tue Nov 1 21:01:09 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2011 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,457 +18,15 @@
package com.cloudera.sqoop.manager;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.io.SplittableBufferedWriter;
-import com.cloudera.sqoop.util.AsyncSink;
-import com.cloudera.sqoop.util.DirectImportUtils;
-import com.cloudera.sqoop.util.ErrorableAsyncSink;
-import com.cloudera.sqoop.util.ErrorableThread;
-import com.cloudera.sqoop.util.Executor;
-import com.cloudera.sqoop.util.ImportException;
-import com.cloudera.sqoop.util.JdbcUrl;
-import com.cloudera.sqoop.util.LoggingAsyncSink;
-import com.cloudera.sqoop.util.PerfCounters;
/**
- * Manages direct dumps from Postgresql databases via psql COPY TO STDOUT
- * commands.
+ * @deprecated Moving to use org.apache.sqoop namespace.
*/
-public class DirectPostgresqlManager extends PostgresqlManager {
- public static final Log LOG = LogFactory.getLog(
- DirectPostgresqlManager.class.getName());
+public class DirectPostgresqlManager
+ extends org.apache.sqoop.manager.DirectPostgresqlManager {
public DirectPostgresqlManager(final SqoopOptions opts) {
- // Inform superclass that we're overriding import method via alt.
- // constructor.
- super(opts, true);
- }
-
- private static final String PSQL_CMD = "psql";
-
- /** Copies data directly into HDFS, adding the user's chosen line terminator
- char to each record.
- */
- static class PostgresqlAsyncSink extends ErrorableAsyncSink {
- private final SplittableBufferedWriter writer;
- private final PerfCounters counters;
- private final SqoopOptions options;
-
- PostgresqlAsyncSink(final SplittableBufferedWriter w,
- final SqoopOptions opts, final PerfCounters ctrs) {
- this.writer = w;
- this.options = opts;
- this.counters = ctrs;
- }
-
- public void processStream(InputStream is) {
- child = new PostgresqlStreamThread(is, writer, options, counters);
- child.start();
- }
-
- private static class PostgresqlStreamThread extends ErrorableThread {
- public static final Log LOG = LogFactory.getLog(
- PostgresqlStreamThread.class.getName());
-
- private final SplittableBufferedWriter writer;
- private final InputStream stream;
- private final SqoopOptions options;
- private final PerfCounters counters;
-
- PostgresqlStreamThread(final InputStream is,
- final SplittableBufferedWriter w,
- final SqoopOptions opts, final PerfCounters ctrs) {
- this.stream = is;
- this.writer = w;
- this.options = opts;
- this.counters = ctrs;
- }
-
- public void run() {
- BufferedReader r = null;
- SplittableBufferedWriter w = this.writer;
-
- char recordDelim = this.options.getOutputRecordDelim();
-
- try {
- r = new BufferedReader(new InputStreamReader(this.stream));
-
- // read/write transfer loop here.
- while (true) {
- String inLine = r.readLine();
- if (null == inLine) {
- break; // EOF
- }
-
- w.write(inLine);
- w.write(recordDelim);
- w.allowSplit();
- counters.addBytes(1 + inLine.length());
- }
- } catch (IOException ioe) {
- LOG.error("IOException reading from psql: " + ioe.toString());
- // set the error bit so our caller can see that something went wrong.
- setError();
- } finally {
- if (null != r) {
- try {
- r.close();
- } catch (IOException ioe) {
- LOG.info("Error closing FIFO stream: " + ioe.toString());
- }
- }
-
- if (null != w) {
- try {
- w.close();
- } catch (IOException ioe) {
- LOG.info("Error closing HDFS stream: " + ioe.toString());
- }
- }
- }
- }
- }
- }
-
- /**
- * Takes a list of columns and turns them into a string like
- * "col1, col2, col3...".
- */
- private String getColumnListStr(String [] cols) {
- if (null == cols) {
- return null;
- }
-
- StringBuilder sb = new StringBuilder();
- boolean first = true;
- for (String col : cols) {
- if (!first) {
- sb.append(", ");
- }
- sb.append(col);
- first = false;
- }
-
- return sb.toString();
- }
-
- /**
- * @return the Postgresql-specific SQL command to copy the
- * table ("COPY .... TO STDOUT").
- */
- private String getCopyCommand(String tableName) {
-
- // Format of this command is:
- //
- // COPY table(col, col....) TO STDOUT
- // or COPY ( query ) TO STDOUT
- // WITH DELIMITER 'fieldsep'
- // CSV
- // QUOTE 'quotechar'
- // ESCAPE 'escapechar'
- // FORCE QUOTE col, col, col....
-
- StringBuilder sb = new StringBuilder();
- String [] cols = getColumnNames(tableName);
-
- String escapedTableName = escapeTableName(tableName);
-
- sb.append("COPY ");
- String whereClause = this.options.getWhereClause();
- if (whereClause != null && whereClause.length() > 0) {
- // Import from a SELECT QUERY
- sb.append("(");
- sb.append("SELECT ");
- if (null != cols) {
- sb.append(getColumnListStr(cols));
- } else {
- sb.append("*");
- }
-
- sb.append(" FROM ");
- sb.append(escapedTableName);
- sb.append(" WHERE ");
- sb.append(whereClause);
- sb.append(")");
- } else {
- // Import just the table.
- sb.append(escapedTableName);
- if (null != cols) {
- // specify columns.
- sb.append("(");
- sb.append(getColumnListStr(cols));
- sb.append(")");
- }
- }
-
- // Translate delimiter characters to '\ooo' octal representation.
- sb.append(" TO STDOUT WITH DELIMITER E'\\");
- sb.append(Integer.toString((int) this.options.getOutputFieldDelim(), 8));
- sb.append("' CSV ");
- if (this.options.getOutputEnclosedBy() != '\0') {
- sb.append("QUOTE E'\\");
- sb.append(Integer.toString((int) this.options.getOutputEnclosedBy(), 8));
- sb.append("' ");
- }
- if (this.options.getOutputEscapedBy() != '\0') {
- sb.append("ESCAPE E'\\");
- sb.append(Integer.toString((int) this.options.getOutputEscapedBy(), 8));
- sb.append("' ");
- }
-
- // add the "FORCE QUOTE col, col, col..." clause if quotes are required.
- if (null != cols && this.options.isOutputEncloseRequired()) {
- sb.append("FORCE QUOTE ");
- sb.append(getColumnListStr(cols));
- }
-
- sb.append(";");
-
- String copyCmd = sb.toString();
- LOG.debug("Copy command is " + copyCmd);
- return copyCmd;
- }
-
- /** Write the COPY command to a temp file.
- * @return the filename we wrote to.
- */
- private String writeCopyCommand(String command) throws IOException {
- String tmpDir = options.getTempDir();
- File tempFile = File.createTempFile("tmp-", ".sql", new File(tmpDir));
- BufferedWriter w = new BufferedWriter(
- new OutputStreamWriter(new FileOutputStream(tempFile)));
- w.write(command);
- w.newLine();
- w.close();
- return tempFile.toString();
- }
-
- /** Write the user's password to a file that is chmod 0600.
- @return the filename.
- */
- private String writePasswordFile(String password) throws IOException {
-
- String tmpDir = options.getTempDir();
- File tempFile = File.createTempFile("pgpass", ".pgpass", new File(tmpDir));
- LOG.debug("Writing password to tempfile: " + tempFile);
-
- // Make sure it's only readable by the current user.
- DirectImportUtils.setFilePermissions(tempFile, "0600");
-
- // Actually write the password data into the file.
- BufferedWriter w = new BufferedWriter(
- new OutputStreamWriter(new FileOutputStream(tempFile)));
- w.write("*:*:*:*:" + password);
- w.close();
- return tempFile.toString();
- }
-
- // TODO(aaron): Refactor this method to be much shorter.
- // CHECKSTYLE:OFF
- @Override
- /**
- * Import the table into HDFS by using psql to pull the data out of the db
- * via COPY FILE TO STDOUT.
- */
- public void importTable(ImportJobContext context)
- throws IOException, ImportException {
-
- String tableName = context.getTableName();
- SqoopOptions options = context.getOptions();
-
- LOG.info("Beginning psql fast path import");
-
- if (options.getFileLayout() != SqoopOptions.FileLayout.TextFile) {
- // TODO(aaron): Support SequenceFile-based load-in
- LOG.warn("File import layout" + options.getFileLayout()
- + " is not supported by");
- LOG.warn("Postgresql direct import; import will proceed as text files.");
- }
-
- String commandFilename = null;
- String passwordFilename = null;
- Process p = null;
- AsyncSink sink = null;
- AsyncSink errSink = null;
- PerfCounters counters = new PerfCounters();
-
- try {
- // Get the COPY TABLE command to issue, write this to a file, and pass
- // it in to psql with -f filename. Then make sure we delete this file
- // in our finally block.
- String copyCmd = getCopyCommand(tableName);
- commandFilename = writeCopyCommand(copyCmd);
-
- // Arguments to pass to psql on the command line.
- ArrayList<String> args = new ArrayList<String>();
-
- // Environment to pass to psql.
- List<String> envp = Executor.getCurEnvpStrings();
-
- // We need to parse the connect string URI to determine the database
- // name and the host and port. If the host is localhost and the port is
- // not specified, we don't want to pass this to psql, because we want to
- // force the use of a UNIX domain socket, not a TCP/IP socket.
- String connectString = options.getConnectString();
- String databaseName = JdbcUrl.getDatabaseName(connectString);
- String hostname = JdbcUrl.getHostName(connectString);
- int port = JdbcUrl.getPort(connectString);
-
- if (null == databaseName) {
- throw new ImportException("Could not determine database name");
- }
-
- LOG.info("Performing import of table " + tableName + " from database "
- + databaseName);
- args.add(PSQL_CMD); // requires that this is on the path.
- args.add("--tuples-only");
- args.add("--quiet");
-
- String username = options.getUsername();
- if (username != null) {
- args.add("--username");
- args.add(username);
- String password = options.getPassword();
- if (null != password) {
- passwordFilename = writePasswordFile(password);
- // Need to send PGPASSFILE environment variable specifying
- // location of our postgres file.
- envp.add("PGPASSFILE=" + passwordFilename);
- }
- }
-
- args.add("--host");
- args.add(hostname);
-
- if (port != -1) {
- args.add("--port");
- args.add(Integer.toString(port));
- }
-
- if (null != databaseName && databaseName.length() > 0) {
- args.add(databaseName);
- }
-
- // The COPY command is in a script file.
- args.add("-f");
- args.add(commandFilename);
-
- // begin the import in an external process.
- LOG.debug("Starting psql with arguments:");
- for (String arg : args) {
- LOG.debug(" " + arg);
- }
-
- // This writer will be closed by AsyncSink.
- SplittableBufferedWriter w = DirectImportUtils.createHdfsSink(
- options.getConf(), options, context);
-
- // Actually start the psql dump.
- p = Runtime.getRuntime().exec(args.toArray(new String[0]),
- envp.toArray(new String[0]));
-
- // read from the stdout pipe into the HDFS writer.
- InputStream is = p.getInputStream();
- sink = new PostgresqlAsyncSink(w, options, counters);
-
- LOG.debug("Starting stream sink");
- counters.startClock();
- sink.processStream(is);
- errSink = new LoggingAsyncSink(LOG);
- errSink.processStream(p.getErrorStream());
- } finally {
- // block until the process is done.
- LOG.debug("Waiting for process completion");
- int result = 0;
- if (null != p) {
- while (true) {
- try {
- result = p.waitFor();
- } catch (InterruptedException ie) {
- // interrupted; loop around.
- continue;
- }
-
- break;
- }
- }
-
- // Remove any password file we wrote
- if (null != passwordFilename) {
- if (!new File(passwordFilename).delete()) {
- LOG.error("Could not remove postgresql password file "
- + passwordFilename);
- LOG.error("You should remove this file to protect your credentials.");
- }
- }
-
- if (null != commandFilename) {
- // We wrote the COPY comand to a tmpfile. Remove it.
- if (!new File(commandFilename).delete()) {
- LOG.info("Could not remove temp file: " + commandFilename);
- }
- }
-
- // block until the stream sink is done too.
- int streamResult = 0;
- if (null != sink) {
- while (true) {
- try {
- streamResult = sink.join();
- } catch (InterruptedException ie) {
- // interrupted; loop around.
- continue;
- }
-
- break;
- }
- }
-
- // Attempt to block for stderr stream sink; errors are advisory.
- if (null != errSink) {
- try {
- if (0 != errSink.join()) {
- LOG.info("Encountered exception reading stderr stream");
- }
- } catch (InterruptedException ie) {
- LOG.info("Thread interrupted waiting for stderr to complete: "
- + ie.toString());
- }
- }
-
- LOG.info("Transfer loop complete.");
-
- if (0 != result) {
- throw new IOException("psql terminated with status "
- + Integer.toString(result));
- }
-
- if (0 != streamResult) {
- throw new IOException("Encountered exception in stream sink");
- }
-
- counters.stopClock();
- LOG.info("Transferred " + counters.toString());
- }
- }
-
- @Override
- public boolean supportsStagingForExport() {
- return false;
+ super(opts);
}
- // CHECKSTYLE:ON
}
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ExportJobContext.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ExportJobContext.java?rev=1196272&r1=1196271&r2=1196272&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ExportJobContext.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ExportJobContext.java Tue Nov 1 21:01:09 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2011 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -23,55 +21,18 @@ package com.cloudera.sqoop.manager;
import com.cloudera.sqoop.SqoopOptions;
/**
- * A set of parameters describing an export operation; this is passed to
- * ConnManager.exportTable() as its argument.
+ * @deprecated Moving to use org.apache.sqoop namespace.
*/
-public class ExportJobContext {
-
- private String tableName;
- private String jarFile;
- private SqoopOptions options;
- private ConnManager manager;
+public class ExportJobContext
+ extends org.apache.sqoop.manager.ExportJobContext {
public ExportJobContext(final String table, final String jar,
final SqoopOptions opts) {
- this.tableName = table;
- this.jarFile = jar;
- this.options = opts;
- }
-
- /** @return the name of the table to export. */
- public String getTableName() {
- return tableName;
- }
-
- /** @return the name of the jar file containing the user's compiled
- * ORM classes to use during the export.
- */
- public String getJarFile() {
- return jarFile;
+ super(table, jar, opts);
}
- /** @return the SqoopOptions configured by the user */
- public SqoopOptions getOptions() {
- return options;
- }
-
- /**
- * Set the ConnManager instance to be used during the export's
- * configuration.
- */
- public void setConnManager(ConnManager mgr) {
- this.manager = mgr;
- }
-
- /**
- * Get the ConnManager instance to use during an export's
- * configuration stage.
- */
public ConnManager getConnManager() {
- return this.manager;
+ return (ConnManager)super.getConnManager();
}
-
}
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/GenericJdbcManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/GenericJdbcManager.java?rev=1196272&r1=1196271&r2=1196272&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/GenericJdbcManager.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/GenericJdbcManager.java Tue Nov 1 21:01:09 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2011 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,67 +18,17 @@
package com.cloudera.sqoop.manager;
-import java.sql.Connection;
-import java.sql.SQLException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import com.cloudera.sqoop.SqoopOptions;
/**
- * Database manager that is connects to a generic JDBC-compliant
- * database; its constructor is parameterized on the JDBC Driver
- * class to load.
+ * @deprecated Moving to use org.apache.sqoop namespace.
*/
-public class GenericJdbcManager extends SqlManager {
-
- public static final Log LOG = LogFactory.getLog(
- GenericJdbcManager.class.getName());
-
- private String jdbcDriverClass;
- private Connection connection;
+public class GenericJdbcManager
+ extends org.apache.sqoop.manager.GenericJdbcManager {
public GenericJdbcManager(final String driverClass, final SqoopOptions opts) {
- super(opts);
-
- this.jdbcDriverClass = driverClass;
- }
-
- @Override
- public Connection getConnection() throws SQLException {
- if (null == this.connection) {
- this.connection = makeConnection();
- }
-
- return this.connection;
+ super(driverClass, opts);
}
- protected boolean hasOpenConnection() {
- return this.connection != null;
- }
-
- /**
- * Any reference to the connection managed by this manager is nulled.
- * If doClose is true, then this method will attempt to close the
- * connection first.
- * @param doClose if true, try to close the connection before forgetting it.
- */
- protected void discardConnection(boolean doClose) throws SQLException {
- if (doClose && hasOpenConnection()) {
- this.connection.close();
- }
-
- this.connection = null;
- }
-
- public void close() throws SQLException {
- super.close();
- discardConnection(true);
- }
-
- public String getDriverClass() {
- return jdbcDriverClass;
- }
}
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/HsqldbManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/HsqldbManager.java?rev=1196272&r1=1196271&r2=1196272&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/HsqldbManager.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/HsqldbManager.java Tue Nov 1 21:01:09 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2011 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,69 +18,16 @@
package com.cloudera.sqoop.manager;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.mapreduce.AsyncSqlOutputFormat;
-
-import com.cloudera.sqoop.util.ExportException;
-
/**
- * Manages connections to hsqldb databases.
- * Extends generic SQL manager.
+ * @deprecated Moving to use org.apache.sqoop namespace.
*/
-public class HsqldbManager extends GenericJdbcManager {
-
- public static final Log LOG = LogFactory.getLog(
- HsqldbManager.class.getName());
-
- // driver class to ensure is loaded when making db connection.
- private static final String DRIVER_CLASS = "org.hsqldb.jdbcDriver";
-
- // HsqlDb doesn't have a notion of multiple "databases"; the user's database
- // is always called "PUBLIC".
- private static final String HSQL_SCHEMA_NAME = "PUBLIC";
+public class HsqldbManager
+ extends org.apache.sqoop.manager.HsqldbManager {
public HsqldbManager(final SqoopOptions opts) {
- super(DRIVER_CLASS, opts);
+ super(opts);
}
- /**
- * Return list of databases hosted by the server.
- * HSQLDB only supports a single schema named "PUBLIC".
- */
- @Override
- public String[] listDatabases() {
- String [] databases = {HSQL_SCHEMA_NAME};
- return databases;
- }
-
- @Override
- /**
- * {@inheritDoc}
- */
- protected String getCurTimestampQuery() {
- // HSQLDB requires that you select from a table; this table is
- // guaranteed to exist.
- return "SELECT CURRENT_TIMESTAMP FROM INFORMATION_SCHEMA.SYSTEM_TABLES";
- }
-
- @Override
- public boolean supportsStagingForExport() {
- return true;
- }
-
- @Override
- /** {@inheritDoc} */
- public void exportTable(ExportJobContext context)
- throws IOException, ExportException {
- // HSQLDB does not support multi-row inserts; disable that before export.
- context.getOptions().getConf().setInt(
- AsyncSqlOutputFormat.RECORDS_PER_STATEMENT_KEY, 1);
- super.exportTable(context);
- }
}
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ImportJobContext.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ImportJobContext.java?rev=1196272&r1=1196271&r2=1196272&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ImportJobContext.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ImportJobContext.java Tue Nov 1 21:01:09 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2011 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,83 +18,23 @@
package com.cloudera.sqoop.manager;
-import org.apache.hadoop.mapreduce.InputFormat;
-import com.cloudera.sqoop.mapreduce.db.DataDrivenDBInputFormat;
-import com.cloudera.sqoop.SqoopOptions;
import org.apache.hadoop.fs.Path;
+import com.cloudera.sqoop.SqoopOptions;
+
/**
- * A set of parameters describing an import operation; this is passed to
- * ConnManager.importTable() as its argument.
+ * @deprecated Moving to use org.apache.sqoop namespace.
*/
-public class ImportJobContext {
-
- private String tableName;
- private String jarFile;
- private SqoopOptions options;
- private Class<? extends InputFormat> inputFormatClass;
- private Path destination;
- private ConnManager manager;
+public class ImportJobContext
+ extends org.apache.sqoop.manager.ImportJobContext {
public ImportJobContext(final String table, final String jar,
final SqoopOptions opts, final Path destination) {
- this.tableName = table;
- this.jarFile = jar;
- this.options = opts;
- this.inputFormatClass = DataDrivenDBInputFormat.class;
- this.destination = destination;
- }
-
- /** @return the name of the table to import. */
- public String getTableName() {
- return tableName;
- }
-
- /** @return the name of the jar file containing the user's compiled
- * ORM classes to use during the import.
- */
- public String getJarFile() {
- return jarFile;
- }
-
- /** @return the SqoopOptions configured by the user */
- public SqoopOptions getOptions() {
- return options;
- }
-
- /** Set the InputFormat to use for the import job. */
- public void setInputFormat(Class<? extends InputFormat> ifClass) {
- this.inputFormatClass = ifClass;
- }
-
- /** @return the InputFormat to use for the import job. */
- public Class<? extends InputFormat> getInputFormat() {
- return this.inputFormatClass;
+ super(table, jar, opts, destination);
}
- /**
- * @return the destination path to where the output files will
- * be first saved.
- */
- public Path getDestination() {
- return this.destination;
- }
-
- /**
- * Set the ConnManager instance to be used during the import's
- * configuration.
- */
- public void setConnManager(ConnManager mgr) {
- this.manager = mgr;
- }
-
- /**
- * Get the ConnManager instance to use during an import's
- * configuration stage.
- */
public ConnManager getConnManager() {
- return this.manager;
+ return (ConnManager)super.getConnManager();
}
-
}
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/InformationSchemaManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/InformationSchemaManager.java?rev=1196272&r1=1196271&r2=1196272&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/InformationSchemaManager.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/InformationSchemaManager.java Tue Nov 1 21:01:09 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2011 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,54 +18,17 @@
package com.cloudera.sqoop.manager;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import com.cloudera.sqoop.SqoopOptions;
/**
- * Database manager that queries "information schema" directly
- * (instead of metadata calls) to retrieve information.
+ * @deprecated Moving to use org.apache.sqoop namespace.
*/
-public abstract class InformationSchemaManager extends CatalogQueryManager {
-
- public static final Log LOG = LogFactory.getLog(
- InformationSchemaManager.class.getName());
+public abstract class InformationSchemaManager
+ extends org.apache.sqoop.manager.InformationSchemaManager {
public InformationSchemaManager(final String driverClass,
final SqoopOptions opts) {
super(driverClass, opts);
}
-
- protected abstract String getSchemaQuery();
-
- @Override
- protected String getListTablesQuery() {
- return
- "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES "
- + "WHERE TABLE_SCHEMA = (" + getSchemaQuery() + ")";
- }
-
- @Override
- protected String getListColumnsQuery(String tableName) {
- return
- "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS "
- + "WHERE TABLE_SCHEMA = (" + getSchemaQuery() + ") "
- + " AND TABLE_NAME = '" + tableName + "' ";
- }
-
- @Override
- protected String getPrimaryKeyQuery(String tableName) {
- return
- "SELECT kcu.COLUMN_NAME FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc, "
- + " INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu "
- + "WHERE tc.TABLE_SCHEMA = kcu.TABLE_SCHEMA "
- + " AND tc.TABLE_NAME = kcu.TABLE_NAME "
- + " AND tc.CONSTRAINT_SCHEMA = kcu.CONSTRAINT_SCHEMA "
- + " AND tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME "
- + " AND tc.TABLE_SCHEMA = (" + getSchemaQuery() + ") "
- + " AND tc.TABLE_NAME = '" + tableName + "' "
- + " AND tc.CONSTRAINT_TYPE = 'PRIMARY KEY'";
- }
}
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ManagerFactory.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ManagerFactory.java?rev=1196272&r1=1196271&r2=1196272&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ManagerFactory.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/ManagerFactory.java Tue Nov 1 21:01:09 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2011 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,35 +18,10 @@
package com.cloudera.sqoop.manager;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.metastore.JobData;
-
/**
- * Interface for factory classes for ConnManager implementations.
- * ManagerFactories are instantiated by o.a.h.s.ConnFactory and
- * stored in an ordered list. The ConnFactory.getManager() implementation
- * calls the accept() method of each ManagerFactory, in order until
- * one such call returns a non-null ConnManager instance.
+ * @deprecated Moving to use org.apache.sqoop namespace.
*/
-public abstract class ManagerFactory {
- @Deprecated
- /** Do not use accept(SqoopOptions). Use accept(JobData) instead. */
- public ConnManager accept(SqoopOptions options) {
- throw new RuntimeException(
- "Deprecated method; override ManagerFactory.accept(JobData)");
- }
-
- /**
- * Instantiate a ConnManager that can fulfill the database connection
- * requirements of the task specified in the JobData.
- * @param jobData the user-provided arguments that configure this
- * Sqoop job.
- * @return a ConnManager that can connect to the specified database
- * and perform the operations required, or null if this factory cannot
- * find a suitable ConnManager implementation.
- */
- public ConnManager accept(JobData jobData) {
- return accept(jobData.getSqoopOptions());
- }
+public abstract class ManagerFactory
+ extends org.apache.sqoop.manager.ManagerFactory {
}
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/MySQLManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/MySQLManager.java?rev=1196272&r1=1196271&r2=1196272&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/MySQLManager.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/MySQLManager.java Tue Nov 1 21:01:09 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2011 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,210 +18,17 @@
package com.cloudera.sqoop.manager;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.hadoop.util.StringUtils;
import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.util.ImportException;
/**
- * Manages connections to MySQL databases.
+ * @deprecated Moving to use org.apache.sqoop namespace.
*/
-public class MySQLManager extends InformationSchemaManager {
-
- public static final Log LOG = LogFactory.getLog(MySQLManager.class.getName());
-
- // driver class to ensure is loaded when making db connection.
- private static final String DRIVER_CLASS = "com.mysql.jdbc.Driver";
-
- // set to true after we warn the user that we can use direct fastpath.
- private static boolean warningPrinted = false;
+public class MySQLManager
+ extends org.apache.sqoop.manager.MySQLManager {
public MySQLManager(final SqoopOptions opts) {
- super(DRIVER_CLASS, opts);
- }
-
- @Override
- protected void initOptionDefaults() {
- if (options.getFetchSize() == null) {
- LOG.info("Preparing to use a MySQL streaming resultset.");
- options.setFetchSize(Integer.MIN_VALUE);
- } else if (
- !options.getFetchSize().equals(Integer.MIN_VALUE)
- && !options.getFetchSize().equals(0)) {
- LOG.info("Argument '--fetch-size " + options.getFetchSize()
- + "' will probably get ignored by MySQL JDBC driver.");
- // see also
- // http://dev.mysql.com/doc/refman/5.5/en
- // /connector-j-reference-implementation-notes.html
- }
+ super(opts);
}
- @Override
- protected String getColNamesQuery(String tableName) {
- // Use mysql-specific hints and LIMIT to return fast
- return "SELECT t.* FROM " + escapeTableName(tableName) + " AS t LIMIT 1";
- }
-
- @Override
- public void importTable(ImportJobContext context)
- throws IOException, ImportException {
-
- // Check that we're not doing a MapReduce from localhost. If we are, point
- // out that we could use mysqldump.
- if (!MySQLManager.warningPrinted) {
- String connectString = context.getOptions().getConnectString();
-
- if (null != connectString) {
- // DirectMySQLManager will probably be faster.
- LOG.warn("It looks like you are importing from mysql.");
- LOG.warn("This transfer can be faster! Use the --direct");
- LOG.warn("option to exercise a MySQL-specific fast path.");
-
- MySQLManager.markWarningPrinted(); // don't display this twice.
- }
- }
-
- checkDateTimeBehavior(context);
-
- // Then run the normal importTable() method.
- super.importTable(context);
- }
-
- /**
- * Set a flag to prevent printing the --direct warning twice.
- */
- protected static void markWarningPrinted() {
- MySQLManager.warningPrinted = true;
- }
-
- /**
- * MySQL allows TIMESTAMP fields to have the value '0000-00-00 00:00:00',
- * which causes errors in import. If the user has not set the
- * zeroDateTimeBehavior property already, we set it for them to coerce
- * the type to null.
- */
- private void checkDateTimeBehavior(ImportJobContext context) {
- final String ZERO_BEHAVIOR_STR = "zeroDateTimeBehavior";
- final String CONVERT_TO_NULL = "=convertToNull";
-
- String connectStr = context.getOptions().getConnectString();
- if (connectStr.indexOf("jdbc:") != 0) {
- // This connect string doesn't have the prefix we expect.
- // We can't parse the rest of it here.
- return;
- }
-
- // This starts with 'jdbc:mysql://' ... let's remove the 'jdbc:'
- // prefix so that java.net.URI can parse the rest of the line.
- String uriComponent = connectStr.substring(5);
- try {
- URI uri = new URI(uriComponent);
- String query = uri.getQuery(); // get the part after a '?'
-
- // If they haven't set the zeroBehavior option, set it to
- // squash-null for them.
- if (null == query) {
- connectStr = connectStr + "?" + ZERO_BEHAVIOR_STR + CONVERT_TO_NULL;
- LOG.info("Setting zero DATETIME behavior to convertToNull (mysql)");
- } else if (query.length() == 0) {
- connectStr = connectStr + ZERO_BEHAVIOR_STR + CONVERT_TO_NULL;
- LOG.info("Setting zero DATETIME behavior to convertToNull (mysql)");
- } else if (query.indexOf(ZERO_BEHAVIOR_STR) == -1) {
- if (!connectStr.endsWith("&")) {
- connectStr = connectStr + "&";
- }
- connectStr = connectStr + ZERO_BEHAVIOR_STR + CONVERT_TO_NULL;
- LOG.info("Setting zero DATETIME behavior to convertToNull (mysql)");
- }
-
- LOG.debug("Rewriting connect string to " + connectStr);
- context.getOptions().setConnectString(connectStr);
- } catch (URISyntaxException use) {
- // Just ignore this. If we can't parse the URI, don't attempt
- // to add any extra flags to it.
- LOG.debug("mysql: Couldn't parse connect str in checkDateTimeBehavior: "
- + use);
- }
- }
-
- @Override
- public void execAndPrint(String s) {
- // Override default execAndPrint() with a special version that forces
- // use of fully-buffered ResultSets (MySQLManager uses streaming ResultSets
- // in the default execute() method; but the execAndPrint() method needs to
- // issue overlapped queries for metadata.)
-
- ResultSet results = null;
- try {
- // Explicitly setting fetchSize to zero disables streaming.
- results = super.execute(s, 0);
- } catch (SQLException sqlE) {
- LOG.error("Error executing statement: "
- + StringUtils.stringifyException(sqlE));
- release();
- return;
- }
-
- PrintWriter pw = new PrintWriter(System.out, true);
- try {
- formatAndPrintResultSet(results, pw);
- } finally {
- pw.close();
- }
- }
-
- /**
- * When using a column name in a generated SQL query, how (if at all)
- * should we escape that column name? e.g., a column named "table"
- * may need to be quoted with backtiks: "`table`".
- *
- * @param colName the column name as provided by the user, etc.
- * @return how the column name should be rendered in the sql text.
- */
- public String escapeColName(String colName) {
- if (null == colName) {
- return null;
- }
- return "`" + colName + "`";
- }
-
- /**
- * When using a table name in a generated SQL query, how (if at all)
- * should we escape that column name? e.g., a table named "table"
- * may need to be quoted with backtiks: "`table`".
- *
- * @param tableName the table name as provided by the user, etc.
- * @return how the table name should be rendered in the sql text.
- */
- public String escapeTableName(String tableName) {
- if (null == tableName) {
- return null;
- }
- return "`" + tableName + "`";
- }
-
- @Override
- public boolean supportsStagingForExport() {
- return true;
- }
-
- @Override
- protected String getListDatabasesQuery() {
- return "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA";
- }
-
- @Override
- protected String getSchemaQuery() {
- return "SELECT SCHEMA()";
- }
}
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/MySQLUtils.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/MySQLUtils.java?rev=1196272&r1=1196271&r2=1196272&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/MySQLUtils.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/MySQLUtils.java Tue Nov 1 21:01:09 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2011 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -20,103 +18,54 @@
package com.cloudera.sqoop.manager;
-import static com.cloudera.sqoop.lib.DelimiterSet.NULL_CHAR;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.OutputStreamWriter;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import com.cloudera.sqoop.config.ConfigurationConstants;
-import com.cloudera.sqoop.config.ConfigurationHelper;
-import com.cloudera.sqoop.util.DirectImportUtils;
-
/**
- * Helper methods and constants for MySQL imports/exports.
+ * @deprecated Moving to use org.apache.sqoop namespace.
*/
public final class MySQLUtils {
private MySQLUtils() {
}
- public static final Log LOG = LogFactory.getLog(MySQLUtils.class.getName());
-
- public static final String MYSQL_DUMP_CMD = "mysqldump";
- public static final String MYSQL_IMPORT_CMD = "mysqlimport";
-
+ public static final String MYSQL_DUMP_CMD =
+ org.apache.sqoop.manager.MySQLUtils.MYSQL_DUMP_CMD;
+ public static final String MYSQL_IMPORT_CMD =
+ org.apache.sqoop.manager.MySQLUtils.MYSQL_IMPORT_CMD;
public static final String OUTPUT_FIELD_DELIM_KEY =
- "sqoop.output.field.delim";
+ org.apache.sqoop.manager.MySQLUtils.OUTPUT_FIELD_DELIM_KEY;
public static final String OUTPUT_RECORD_DELIM_KEY =
- "sqoop.output.record.delim";
+ org.apache.sqoop.manager.MySQLUtils.OUTPUT_RECORD_DELIM_KEY;
public static final String OUTPUT_ENCLOSED_BY_KEY =
- "sqoop.output.enclosed.by";
+ org.apache.sqoop.manager.MySQLUtils.OUTPUT_ENCLOSED_BY_KEY;
public static final String OUTPUT_ESCAPED_BY_KEY =
- "sqoop.output.escaped.by";
+ org.apache.sqoop.manager.MySQLUtils.OUTPUT_ESCAPED_BY_KEY;
public static final String OUTPUT_ENCLOSE_REQUIRED_KEY =
- "sqoop.output.enclose.required";
+ org.apache.sqoop.manager.MySQLUtils.OUTPUT_ENCLOSE_REQUIRED_KEY;
public static final String TABLE_NAME_KEY =
- ConfigurationHelper.getDbInputTableNameProperty();
+ org.apache.sqoop.manager.MySQLUtils.TABLE_NAME_KEY;
public static final String CONNECT_STRING_KEY =
- ConfigurationHelper.getDbUrlProperty();
+ org.apache.sqoop.manager.MySQLUtils.CONNECT_STRING_KEY;
public static final String USERNAME_KEY =
- ConfigurationHelper.getDbUsernameProperty();
+ org.apache.sqoop.manager.MySQLUtils.USERNAME_KEY;
public static final String PASSWORD_KEY =
- ConfigurationHelper.getDbPasswordProperty();
+ org.apache.sqoop.manager.MySQLUtils.PASSWORD_KEY;
public static final String WHERE_CLAUSE_KEY =
- ConfigurationHelper.getDbInputConditionsProperty();
+ org.apache.sqoop.manager.MySQLUtils.WHERE_CLAUSE_KEY;
public static final String EXTRA_ARGS_KEY =
- "sqoop.mysql.extra.args";
-
- public static final String MYSQL_DEFAULT_CHARSET = "ISO_8859_1";
+ org.apache.sqoop.manager.MySQLUtils.EXTRA_ARGS_KEY;
+ public static final String MYSQL_DEFAULT_CHARSET =
+ org.apache.sqoop.manager.MySQLUtils.MYSQL_DEFAULT_CHARSET;
- /**
- * @return true if the user's output delimiters match those used by mysqldump.
- * fields: ,
- * lines: \n
- * optional-enclose: \'
- * escape: \\
- */
public static boolean outputDelimsAreMySQL(Configuration conf) {
- return ',' == (char) conf.getInt(OUTPUT_FIELD_DELIM_KEY, NULL_CHAR)
- && '\n' == (char) conf.getInt(OUTPUT_RECORD_DELIM_KEY, NULL_CHAR)
- && '\'' == (char) conf.getInt(OUTPUT_ENCLOSED_BY_KEY, NULL_CHAR)
- && '\\' == (char) conf.getInt(OUTPUT_ESCAPED_BY_KEY, NULL_CHAR)
- && !conf.getBoolean(OUTPUT_ENCLOSE_REQUIRED_KEY, false);
+ return org.apache.sqoop.manager.MySQLUtils.outputDelimsAreMySQL(conf);
}
- /**
- * Writes the user's password to a tmp file with 0600 permissions.
- * @return the filename used.
- */
public static String writePasswordFile(Configuration conf)
throws IOException {
- // Create the temp file to hold the user's password.
- String tmpDir = conf.get(
- ConfigurationConstants.PROP_JOB_LOCAL_DIRECTORY, "/tmp/");
- File tempFile = File.createTempFile("mysql-cnf", ".cnf", new File(tmpDir));
-
- // Make the password file only private readable.
- DirectImportUtils.setFilePermissions(tempFile, "0600");
-
- // If we're here, the password file is believed to be ours alone. The
- // inability to set chmod 0600 inside Java is troublesome. We have to
- // trust that the external 'chmod' program in the path does the right
- // thing, and returns the correct exit status. But given our inability to
- // re-read the permissions associated with a file, we'll have to make do
- // with this.
- String password = conf.get(PASSWORD_KEY);
- BufferedWriter w = new BufferedWriter(new OutputStreamWriter(
- new FileOutputStream(tempFile)));
- w.write("[client]\n");
- w.write("password=" + password + "\n");
- w.close();
-
- return tempFile.toString();
+ return org.apache.sqoop.manager.MySQLUtils.writePasswordFile(conf);
}
+
}