You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by bl...@apache.org on 2011/10/08 01:42:01 UTC
svn commit: r1180279 - in /incubator/sqoop/trunk/src:
java/com/cloudera/sqoop/manager/SqlManager.java
java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java
test/com/cloudera/sqoop/TestBoundaryQuery.java
Author: blee
Date: Fri Oct 7 23:42:00 2011
New Revision: 1180279
URL: http://svn.apache.org/viewvc?rev=1180279&view=rev
Log:
SQOOP-359 Import fails with Unknown SQL datatype exception
Modified:
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java
incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java
incubator/sqoop/trunk/src/test/com/cloudera/sqoop/TestBoundaryQuery.java
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java?rev=1180279&r1=1180278&r2=1180279&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java Fri Oct 7 23:42:00 2011
@@ -440,11 +440,19 @@ public abstract class SqlManager extends
}
String splitCol = getSplitColumn(opts, null);
- if (null == splitCol && opts.getNumMappers() > 1) {
- // Can't infer a primary key.
- throw new ImportException("A split-by column must be specified for "
- + "parallel free-form query imports. Please specify one with "
- + "--split-by or perform a sequential import with '-m 1'.");
+ if (splitCol == null) {
+ String boundaryQuery = opts.getBoundaryQuery();
+ if (opts.getNumMappers() > 1) {
+ // Can't infer a primary key.
+ throw new ImportException("A split-by column must be specified for "
+ + "parallel free-form query imports. Please specify one with "
+ + "--split-by or perform a sequential import with '-m 1'.");
+ } else if (boundaryQuery != null && !boundaryQuery.isEmpty()) {
+ // Query import with boundary query and no split column specified
+ throw new ImportException("Using a boundary query for a query based "
+ + "import requires specifying the split by column as well. Please "
+ + "specify a column name using --split-by and try again.");
+ }
}
importer.runImport(null, jarFile, splitCol, opts.getConf());
Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java?rev=1180279&r1=1180278&r2=1180279&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java Fri Oct 7 23:42:00 2011
@@ -171,11 +171,16 @@ public class DataDrivenImportJob extends
String inputBoundingQuery = options.getBoundaryQuery();
if(inputBoundingQuery == null) {
+ inputBoundingQuery =
mgr.getInputBoundsQuery(splitByCol, sanitizedQuery);
- }
- if (inputBoundingQuery == null) {
- inputBoundingQuery = "SELECT MIN(" + splitByCol + "), MAX("
- + splitByCol + ") FROM (" + sanitizedQuery + ") AS t1";
+ if (inputBoundingQuery == null) {
+ if (splitByCol != null) {
+ inputBoundingQuery = "SELECT MIN(" + splitByCol + "), MAX("
+ + splitByCol + ") FROM (" + sanitizedQuery + ") AS t1";
+ } else {
+ inputBoundingQuery = "";
+ }
+ }
}
DataDrivenDBInputFormat.setInput(job, DBWritable.class,
inputQuery, inputBoundingQuery);
Modified: incubator/sqoop/trunk/src/test/com/cloudera/sqoop/TestBoundaryQuery.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/test/com/cloudera/sqoop/TestBoundaryQuery.java?rev=1180279&r1=1180278&r2=1180279&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/test/com/cloudera/sqoop/TestBoundaryQuery.java (original)
+++ incubator/sqoop/trunk/src/test/com/cloudera/sqoop/TestBoundaryQuery.java Fri Oct 7 23:42:00 2011
@@ -46,8 +46,9 @@ public class TestBoundaryQuery extends I
* Create the argv to pass to Sqoop.
* @return the argv as an array of strings.
*/
- protected String [] getArgv(boolean includeHadoopFlags, String boundaryQuery,
- String targetDir) {
+ protected String [] getArgv(boolean includeHadoopFlags, boolean tableImport,
+ String boundaryQuery,
+ String targetDir, String... extraArgs) {
ArrayList<String> args = new ArrayList<String>();
@@ -55,20 +56,29 @@ public class TestBoundaryQuery extends I
CommonArgs.addHadoopFlags(args);
}
- args.add("--table");
- args.add(HsqldbTestServer.getTableName());
- args.add("--split-by");
- args.add("INTFIELD1");
+ if (tableImport) {
+ args.add("--table");
+ args.add(HsqldbTestServer.getTableName());
+ } else {
+ args.add("--query");
+ args.add("SELECT INTFIELD1, INTFIELD2 FROM "
+ + HsqldbTestServer.getTableName() + " WHERE $CONDITIONS");
+ }
args.add("--connect");
args.add(HsqldbTestServer.getUrl());
- args.add("--boundary-query");
- args.add(boundaryQuery);
+ if (boundaryQuery != null) {
+ args.add("--boundary-query");
+ args.add(boundaryQuery);
+ }
args.add("--as-sequencefile");
args.add("--target-dir");
args.add(targetDir);
args.add("--class-name");
args.add(getTableName());
args.add("--verbose");
+ for (String extraArg : extraArgs) {
+ args.add(extraArg);
+ }
return args.toArray(new String[0]);
}
@@ -89,18 +99,18 @@ public class TestBoundaryQuery extends I
return Integer.parseInt(parts[0]);
}
- public void runQueryTest(String query, int numExpectedResults,
- int expectedSum, String targetDir)
- throws IOException {
+ public void runQueryTest(String query, boolean tableImport,
+ int numExpectedResults, int expectedSum, String targetDir,
+ String... extraArgs) throws IOException {
ClassLoader prevClassLoader = null;
SequenceFile.Reader reader = null;
- String [] argv = getArgv(true, query, targetDir);
+ String [] argv = getArgv(true, tableImport, query, targetDir, extraArgs);
runImport(argv);
try {
SqoopOptions opts = new ImportTool().parseArguments(
- getArgv(false, query, targetDir),
+ getArgv(false, tableImport, query, targetDir, extraArgs),
null, null, true);
CompilationManager compileMgr = new CompilationManager(opts);
@@ -161,6 +171,13 @@ public class TestBoundaryQuery extends I
String query = "select min(intfield1), max(intfield1) from "
+ getTableName() +" where intfield1 in (3, 5)";
- runQueryTest(query, 2, 8, getTablePath().toString());
+ runQueryTest(query, true, 2, 8, getTablePath().toString(),
+ "--split-by", "INTFIELD1");
+ }
+
+ public void testNoBoundaryQuerySingleMapper() throws IOException {
+
+ runQueryTest(null, false, 4, 16, getTablePath().toString(),
+ "--m", "1");
}
}