You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ma...@apache.org on 2009/07/31 08:39:10 UTC
svn commit: r799524 - in /hadoop/mapreduce/trunk: ./
src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/
src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/
src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/
Author: matei
Date: Fri Jul 31 06:39:10 2009
New Revision: 799524
URL: http://svn.apache.org/viewvc?rev=799524&view=rev
Log:
MAPREDUCE-816. Rename "local" mysql import to "direct" in Sqoop. (Aaron Kimball via matei).
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ConnFactory.java
hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ImportOptions.java
hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/ConnManager.java
hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/LocalMySQLManager.java
hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/MySQLManager.java
hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/LocalMySQLTest.java
hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/MySQLAuthTest.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=799524&r1=799523&r2=799524&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Jul 31 06:39:10 2009
@@ -49,6 +49,9 @@
IMPROVEMENTS
+ MAPREDUCE-816. Rename "local" mysql import to "direct" in Sqoop.
+ (Aaron Kimball via matei)
+
HADOOP-5967. Sqoop should only use a single map task. (Aaron Kimball via
tomwhite)
Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ConnFactory.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ConnFactory.java?rev=799524&r1=799523&r2=799524&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ConnFactory.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ConnFactory.java Fri Jul 31 06:39:10 2009
@@ -71,7 +71,7 @@
}
if (scheme.equals("jdbc:mysql:")) {
- if (opts.isLocal()) {
+ if (opts.isDirect()) {
return new LocalMySQLManager(opts);
} else {
return new MySQLManager(opts);
Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ImportOptions.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ImportOptions.java?rev=799524&r1=799523&r2=799524&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ImportOptions.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ImportOptions.java Fri Jul 31 06:39:10 2009
@@ -93,7 +93,7 @@
private String driverClassName;
private String warehouseDir;
private FileLayout layout;
- private boolean local; // if true and conn is mysql, use mysqldump.
+ private boolean direct; // if true and conn is mysql, use mysqldump.
private String tmpDir; // where temp data goes; usually /tmp
private String hiveHome;
private boolean hiveImport;
@@ -161,10 +161,10 @@
this.className = props.getProperty("java.classname", this.className);
this.packageName = props.getProperty("java.packagename", this.packageName);
- String localImport = props.getProperty("local.import",
- Boolean.toString(this.local)).toLowerCase();
- this.local = "true".equals(localImport) || "yes".equals(localImport)
- || "1".equals(localImport);
+ String directImport = props.getProperty("direct.import",
+ Boolean.toString(this.direct)).toLowerCase();
+ this.direct = "true".equals(directImport) || "yes".equals(directImport)
+ || "1".equals(directImport);
String hiveImportStr = props.getProperty("hive.import",
Boolean.toString(this.hiveImport)).toLowerCase();
@@ -250,7 +250,7 @@
System.out.println("--username (username) Set authentication username");
System.out.println("--password (password) Set authentication password");
System.out.println("-P Read password from console");
- System.out.println("--local Use local import fast path (mysql only)");
+ System.out.println("--direct Use direct import fast path (mysql only)");
System.out.println("");
System.out.println("Import control options:");
System.out.println("--table (tablename) Table to read");
@@ -418,7 +418,11 @@
} else if (args[i].equals("--all-tables")) {
this.allTables = true;
} else if (args[i].equals("--local")) {
- this.local = true;
+ // TODO(aaron): Remove this after suitable deprecation time period.
+ LOG.warn("--local is deprecated; use --direct instead.");
+ this.direct = true;
+ } else if (args[i].equals("--direct")) {
+ this.direct = true;
} else if (args[i].equals("--username")) {
this.username = args[++i];
if (null == this.password) {
@@ -614,8 +618,8 @@
return password;
}
- public boolean isLocal() {
- return local;
+ public boolean isDirect() {
+ return direct;
}
/**
Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/ConnManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/ConnManager.java?rev=799524&r1=799523&r2=799524&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/ConnManager.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/ConnManager.java Fri Jul 31 06:39:10 2009
@@ -65,7 +65,7 @@
/**
* Execute a SQL statement to read the named set of columns from a table.
- * If columns is null, all columns from the table are read. This is a local
+ * If columns is null, all columns from the table are read. This is a direct
* (non-parallelized) read of the table back to the current client.
* The client is responsible for calling ResultSet.close() when done with the
* returned ResultSet object.
Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/LocalMySQLManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/LocalMySQLManager.java?rev=799524&r1=799523&r2=799524&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/LocalMySQLManager.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/LocalMySQLManager.java Fri Jul 31 06:39:10 2009
@@ -49,9 +49,8 @@
import org.apache.hadoop.util.Shell;
/**
- * Manages local connections to MySQL databases
- * that are local to this machine -- so we can use mysqldump to get
- * really fast dumps.
+ * Manages direct connections to MySQL databases
+ * so we can use mysqldump to get really fast dumps.
*/
public class LocalMySQLManager extends MySQLManager {
@@ -391,7 +390,7 @@
// TODO(aaron): Support SequenceFile-based load-in
LOG.warn("File import layout " + options.getFileLayout()
+ " is not supported by");
- LOG.warn("MySQL local import; import will proceed as text files.");
+ LOG.warn("MySQL direct import; import will proceed as text files.");
}
ArrayList<String> args = new ArrayList<String>();
@@ -461,6 +460,7 @@
args.add("--quick"); // no buffering
// TODO(aaron): Add a flag to allow --lock-tables instead for MyISAM data
args.add("--single-transaction");
+ // TODO(aaron): Add --host and --port arguments to support remote direct connects.
String username = options.getUsername();
if (null != username) {
Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/MySQLManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/MySQLManager.java?rev=799524&r1=799523&r2=799524&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/MySQLManager.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/MySQLManager.java Fri Jul 31 06:39:10 2009
@@ -41,7 +41,7 @@
// driver class to ensure is loaded when making db connection.
private static final String DRIVER_CLASS = "com.mysql.jdbc.Driver";
- // set to true after we warn the user that we can use local fastpath.
+ // set to true after we warn the user that we can use direct fastpath.
private static boolean warningPrinted = false;
public MySQLManager(final ImportOptions opts) {
@@ -49,7 +49,7 @@
}
protected MySQLManager(final ImportOptions opts, boolean ignored) {
- // constructor used by subclasses to avoid the --local warning.
+ // constructor used by subclasses to avoid the --direct warning.
super(DRIVER_CLASS, opts);
}
@@ -102,9 +102,11 @@
if (null != connectString && connectString.indexOf("//localhost") != -1) {
// if we're not doing a remote connection, they should have a LocalMySQLManager.
+ // TODO(aaron): After LocalMySQLManager supports --host/--port, this should
+ // always be issued.
LOG.warn("It looks like you are importing from mysql on");
LOG.warn("localhost. This transfer can be faster! Use the");
- LOG.warn("--local option to exercise a MySQL-specific fast");
+ LOG.warn("--direct option to exercise a MySQL-specific fast");
LOG.warn("path.");
MySQLManager.warningPrinted = true; // don't display this twice.
Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/LocalMySQLTest.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/LocalMySQLTest.java?rev=799524&r1=799523&r2=799524&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/LocalMySQLTest.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/LocalMySQLTest.java Fri Jul 31 06:39:10 2009
@@ -193,7 +193,7 @@
args.add(getWarehouseDir());
args.add("--connect");
args.add(CONNECT_STRING);
- args.add("--local");
+ args.add("--direct");
args.add("--username");
args.add(getCurrentUser());
args.add("--where");
Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/MySQLAuthTest.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/MySQLAuthTest.java?rev=799524&r1=799523&r2=799524&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/MySQLAuthTest.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/MySQLAuthTest.java Fri Jul 31 06:39:10 2009
@@ -144,7 +144,7 @@
args.add(getWarehouseDir());
args.add("--connect");
args.add(AUTH_CONNECT_STRING);
- args.add("--local");
+ args.add("--direct");
args.add("--username");
args.add(AUTH_TEST_USER);
args.add("--password");