You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2013/03/20 22:31:04 UTC
git commit: SQOOP-937: Dont generate ORM files for Direct mode
connectors
Updated Branches:
refs/heads/trunk 86812b853 -> f2ba7eaae
SQOOP-937: Dont generate ORM files for Direct mode connectors
(Venkat Ranganathan via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/f2ba7eaa
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/f2ba7eaa
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/f2ba7eaa
Branch: refs/heads/trunk
Commit: f2ba7eaae25ac23c88651e1d14b9624aa5ded6ca
Parents: 86812b8
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Wed Mar 20 14:30:04 2013 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Wed Mar 20 14:30:04 2013 -0700
----------------------------------------------------------------------
src/java/org/apache/sqoop/manager/ConnManager.java | 15 +++++
.../apache/sqoop/manager/DirectNetezzaManager.java | 5 ++
.../org/apache/sqoop/manager/ExportJobContext.java | 7 ++
.../org/apache/sqoop/manager/ImportJobContext.java | 7 ++
.../org/apache/sqoop/mapreduce/ExportJobBase.java | 10 +++-
.../org/apache/sqoop/mapreduce/ImportJobBase.java | 9 ++-
src/java/org/apache/sqoop/mapreduce/JobBase.java | 1 +
src/java/org/apache/sqoop/orm/ClassWriter.java | 15 ++++-
src/java/org/apache/sqoop/tool/CodeGenTool.java | 16 +++++-
.../com/cloudera/sqoop/orm/TestClassWriter.java | 48 +++++++++++++++
10 files changed, 125 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/manager/ConnManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/ConnManager.java b/src/java/org/apache/sqoop/manager/ConnManager.java
index 1b32dc9..a1ac38e 100644
--- a/src/java/org/apache/sqoop/manager/ConnManager.java
+++ b/src/java/org/apache/sqoop/manager/ConnManager.java
@@ -639,5 +639,20 @@ public abstract class ConnManager {
public String getInputBoundsQuery(String splitByCol, String sanitizedQuery) {
return null;
}
+
+ /**
+ * This method allows the ConnManager to override the generation of ORM
+ * classes if the SQOOP generated classes are not used by it.
+ * A return value of false from this method means that the SQOOP ORM
+ * classes are needed to use with the connector.
+ * A return value of true indicates that the connection manager does not
+ * use the SQOOP ORM classes. For example, in the Direct mode of some of
+ * the connectors, the text files are directly processed by DB specific
+ * facilities without even being passed through the SQOOP process and
+ * in those circumstances, it makes sense to disable the ORM generation.
+ */
+ public boolean isORMFacilitySelfManaged() {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
index 0a1e605..ef98936 100644
--- a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
+++ b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
@@ -246,4 +246,9 @@ public class DirectNetezzaManager extends NetezzaManager {
public boolean supportsStagingForExport() {
return false;
}
+
+ @Override
+ public boolean isORMFacilitySelfManaged() {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/manager/ExportJobContext.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/ExportJobContext.java b/src/java/org/apache/sqoop/manager/ExportJobContext.java
index 5699e2f..2a6f2b5 100644
--- a/src/java/org/apache/sqoop/manager/ExportJobContext.java
+++ b/src/java/org/apache/sqoop/manager/ExportJobContext.java
@@ -18,6 +18,9 @@
package org.apache.sqoop.manager;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.util.Jars;
+
import com.cloudera.sqoop.SqoopOptions;
/**
@@ -35,6 +38,10 @@ public class ExportJobContext {
final SqoopOptions opts) {
this.tableName = table;
this.jarFile = jar;
+ if (this.jarFile == null) {
+ // Set the jarFile to the hadoop core jar file.
+ this.jarFile = Jars.getJarPathForClass(Configuration.class);
+ }
this.options = opts;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/manager/ImportJobContext.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/ImportJobContext.java b/src/java/org/apache/sqoop/manager/ImportJobContext.java
index 09a7abe..354cd15 100644
--- a/src/java/org/apache/sqoop/manager/ImportJobContext.java
+++ b/src/java/org/apache/sqoop/manager/ImportJobContext.java
@@ -21,7 +21,10 @@ package org.apache.sqoop.manager;
import org.apache.hadoop.mapreduce.InputFormat;
import com.cloudera.sqoop.mapreduce.db.DataDrivenDBInputFormat;
import com.cloudera.sqoop.SqoopOptions;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.sqoop.util.Jars;
/**
* A set of parameters describing an import operation; this is passed to
@@ -40,6 +43,10 @@ public class ImportJobContext {
final SqoopOptions opts, final Path destination) {
this.tableName = table;
this.jarFile = jar;
+ if (this.jarFile == null) {
+ // Set the jarFile to the hadoop core jar file.
+ this.jarFile = Jars.getJarPathForClass(Configuration.class);
+ }
this.options = opts;
this.inputFormatClass = DataDrivenDBInputFormat.class;
this.destination = destination;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
index ff84974..1065d0b 100644
--- a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
+++ b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
@@ -321,8 +321,14 @@ public class ExportJobBase extends JobBase {
}
}
- String tableClassName =
- new TableClassName(options).getClassForTable(outputTableName);
+
+ String tableClassName = null;
+ if (!cmgr.isORMFacilitySelfManaged()) {
+ tableClassName =
+ new TableClassName(options).getClassForTable(outputTableName);
+ }
+ // For ORM self managed, we leave the tableClassName to null so that
+ // we don't check for non-existing classes.
String ormJarFile = context.getJarFile();
LOG.info("Beginning export of " + outputTableName);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
index f766532..2465f3f 100644
--- a/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
+++ b/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
@@ -196,9 +196,14 @@ public class ImportJobBase extends JobBase {
} else {
LOG.info("Beginning query import.");
}
+ String tableClassName = null;
+ if (!getContext().getConnManager().isORMFacilitySelfManaged()) {
+ tableClassName =
+ new TableClassName(options).getClassForTable(tableName);
+ }
+ // For ORM self managed, we leave the tableClassName to null so that
+ // we don't check for non-existing classes.
- String tableClassName =
- new TableClassName(options).getClassForTable(tableName);
loadJars(conf, ormJarFile, tableClassName);
Job job = new Job(conf);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/mapreduce/JobBase.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/JobBase.java b/src/java/org/apache/sqoop/mapreduce/JobBase.java
index 4e7723f..0df1156 100644
--- a/src/java/org/apache/sqoop/mapreduce/JobBase.java
+++ b/src/java/org/apache/sqoop/mapreduce/JobBase.java
@@ -220,6 +220,7 @@ public class JobBase {
*/
protected void loadJars(Configuration conf, String ormJarFile,
String tableClassName) throws IOException {
+
boolean isLocal = "local".equals(conf.get("mapreduce.jobtracker.address"))
|| "local".equals(conf.get("mapred.job.tracker"));
if (isLocal) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/orm/ClassWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/orm/ClassWriter.java b/src/java/org/apache/sqoop/orm/ClassWriter.java
index 982e444..0202f7f 100644
--- a/src/java/org/apache/sqoop/orm/ClassWriter.java
+++ b/src/java/org/apache/sqoop/orm/ClassWriter.java
@@ -1064,13 +1064,24 @@ public class ClassWriter {
return cleanedColNames;
}
+ /**
+ * Made this a separate method to overcome the 150 line limit of checkstyle.
+ */
+ private void logORMSelfGenerationMessage() {
+ LOG.info("The connection manager declares that it self manages mapping"
+ + " between records & fields and rows & columns. No class will"
+ + " will be generated.");
+ }
/**
* Generate the ORM code for the class.
*/
public void generate() throws IOException {
Map<String, Integer> columnTypes = getColumnTypes();
-
+ if (connManager.isORMFacilitySelfManaged()) {
+ logORMSelfGenerationMessage();
+ return;
+ }
if (columnTypes == null) {
throw new IOException("No columns to generate for ClassWriter");
}
@@ -1110,7 +1121,6 @@ public class ClassWriter {
}
columnTypes.put(identifier, type);
}
-
// Check that all explicitly mapped columns are present in result set
Properties mapping = options.getMapColumnJava();
if (mapping != null && !mapping.isEmpty()) {
@@ -1207,7 +1217,6 @@ public class ClassWriter {
// ignored because we're closing.
}
}
-
if (null != ostream) {
try {
ostream.close();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/tool/CodeGenTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/CodeGenTool.java b/src/java/org/apache/sqoop/tool/CodeGenTool.java
index 8a4aa42..dd34a97 100644
--- a/src/java/org/apache/sqoop/tool/CodeGenTool.java
+++ b/src/java/org/apache/sqoop/tool/CodeGenTool.java
@@ -71,10 +71,24 @@ public class CodeGenTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
// This code generator is being invoked as part of an import or export
// process, and the user has pre-specified a jar and class to use.
// Don't generate.
+ if (manager.isORMFacilitySelfManaged()) {
+ // No need to generated any ORM. Ignore any jar file given on
+ // command line also.
+ LOG.info("The connection manager declares that it self manages mapping"
+ + " between records & fields and rows & columns. The jar file "
+ + " provided will have no effect");
+ }
LOG.info("Using existing jar: " + existingJar);
return existingJar;
}
-
+ if (manager.isORMFacilitySelfManaged()) {
+ // No need to generated any ORM. Ignore any jar file given on
+ // command line also.
+ LOG.info("The connection manager declares that it self manages mapping"
+ + " between records & fields and rows & columns. No class will"
+ + " will be generated.");
+ return null;
+ }
LOG.info("Beginning code generation");
CompilationManager compileMgr = new CompilationManager(options);
ClassWriter classWriter = new ClassWriter(options, manager, tableName,
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/test/com/cloudera/sqoop/orm/TestClassWriter.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/orm/TestClassWriter.java b/src/test/com/cloudera/sqoop/orm/TestClassWriter.java
index 3b77571..bfb25b0 100644
--- a/src/test/com/cloudera/sqoop/orm/TestClassWriter.java
+++ b/src/test/com/cloudera/sqoop/orm/TestClassWriter.java
@@ -38,6 +38,7 @@ import org.junit.Before;
import org.junit.Test;
import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.TestConnFactory.DummyManager;
import com.cloudera.sqoop.manager.ConnManager;
import com.cloudera.sqoop.testutil.DirUtil;
import com.cloudera.sqoop.testutil.HsqldbTestServer;
@@ -485,4 +486,51 @@ public class TestClassWriter extends TestCase {
}
fail("we shouldn't successfully generate code");
}
+
+ private void runFailedGenerationTest(String [] argv,
+ String classNameToCheck) {
+ File codeGenDirFile = new File(CODE_GEN_DIR);
+ File classGenDirFile = new File(JAR_GEN_DIR);
+
+ try {
+ options = new ImportTool().parseArguments(argv,
+ null, options, true);
+ } catch (Exception e) {
+ LOG.error("Could not parse options: " + e.toString());
+ }
+
+ CompilationManager compileMgr = new CompilationManager(options);
+ ClassWriter writer = new ClassWriter(options, manager,
+ HsqldbTestServer.getTableName(), compileMgr);
+
+ try {
+ writer.generate();
+ compileMgr.compile();
+ fail("ORM class file generation succeeded when it was expected to fail");
+ } catch (IOException ioe) {
+ LOG.error("Got IOException from ORM generation as expected : "
+ + ioe.toString());
+ }
+ }
+ /**
+ * A dummy manager that declares that it ORM is self managed.
+ */
+ public static class DummyDirectManager extends DummyManager {
+ @Override
+ public boolean isORMFacilitySelfManaged() {
+ return true;
+ }
+ }
+
+ @Test
+ public void testNoClassGeneration() throws Exception {
+ manager = new DummyDirectManager();
+ String [] argv = {
+ "--bindir",
+ JAR_GEN_DIR,
+ "--outdir",
+ CODE_GEN_DIR,
+ };
+ runFailedGenerationTest(argv, HsqldbTestServer.getTableName());
+ }
}