You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/04/26 18:05:45 UTC
sqoop git commit: SQOOP-2326: Fix Netezza trunc-string option
handling and unnecessary log directory during imports
Repository: sqoop
Updated Branches:
refs/heads/trunk c92f85574 -> d54dddf0b
SQOOP-2326: Fix Netezza trunc-string option handling and unnecessary log directory during imports
(Venkat Ranganathan via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/d54dddf0
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/d54dddf0
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/d54dddf0
Branch: refs/heads/trunk
Commit: d54dddf0b81d1d196788b4e6422b14b88c2a05ac
Parents: c92f855
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Sun Apr 26 09:05:13 2015 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Sun Apr 26 09:05:13 2015 -0700
----------------------------------------------------------------------
.../sqoop/manager/DirectNetezzaManager.java | 4 +--
.../NetezzaExternalTableImportMapper.java | 26 +-------------------
2 files changed, 3 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d54dddf0/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
index daddb8c..06fa976 100644
--- a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
+++ b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
@@ -293,8 +293,8 @@ public class DirectNetezzaManager extends NetezzaManager {
conf.setBoolean(NETEZZA_CTRL_CHARS_OPT,
cmdLine.hasOption(NETEZZA_CTRL_CHARS_LONG_ARG));
- conf.setBoolean(NETEZZA_CTRL_CHARS_OPT,
- cmdLine.hasOption(NETEZZA_CTRL_CHARS_LONG_ARG));
+ conf.setBoolean(NETEZZA_TRUNC_STRING_OPT,
+ cmdLine.hasOption(NETEZZA_TRUNC_STRING_LONG_ARG));
// Always true for Netezza direct mode access
conf.setBoolean(NETEZZA_DATASLICE_ALIGNED_ACCESS_OPT, true);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d54dddf0/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java b/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java
index 486ba7c..939e358 100644
--- a/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java
+++ b/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java
@@ -67,8 +67,6 @@ public abstract class NetezzaExternalTableImportMapper<K, V> extends
.getLog(NetezzaExternalTableImportMapper.class.getName());
private NetezzaJDBCStatementRunner extTableThread;
private PerfCounters counter;
- private String localLogDir = null;
- private String logDir = null;
private File taskAttemptDir = null;
private String getSqlStatement(int myId) throws IOException {
@@ -78,10 +76,6 @@ public abstract class NetezzaExternalTableImportMapper<K, V> extends
String nullValue = conf.get(DirectNetezzaManager.NETEZZA_NULL_VALUE);
- boolean ctrlChars =
- conf.getBoolean(DirectNetezzaManager.NETEZZA_CTRL_CHARS_OPT, false);
- boolean truncString =
- conf.getBoolean(DirectNetezzaManager.NETEZZA_TRUNC_STRING_OPT, false);
int errorThreshold = conf.getInt(
DirectNetezzaManager.NETEZZA_ERROR_THRESHOLD_OPT, 1);
@@ -95,12 +89,6 @@ public abstract class NetezzaExternalTableImportMapper<K, V> extends
sqlStmt.append("' USING (REMOTESOURCE 'JDBC' ");
sqlStmt.append(" BOOLSTYLE 'T_F' ");
sqlStmt.append(" CRINSTRING FALSE ");
- if (ctrlChars) {
- sqlStmt.append(" CTRLCHARS TRUE ");
- }
- if (truncString) {
- sqlStmt.append(" TRUNCSTRING TRUE ");
- }
sqlStmt.append(" DELIMITER ");
sqlStmt.append(Integer.toString(fd));
sqlStmt.append(" ENCODING 'internal' ");
@@ -131,13 +119,6 @@ public abstract class NetezzaExternalTableImportMapper<K, V> extends
sqlStmt.append(" MAXERRORS ").append(errorThreshold);
- File logDirPath = new File(taskAttemptDir, localLogDir);
- logDirPath.mkdirs();
- if (logDirPath.canWrite() && logDirPath.isDirectory()) {
- sqlStmt.append(" LOGDIR ").append(logDirPath.getAbsolutePath()).append(' ');
- } else {
- throw new IOException("Unable to create log directory specified");
- }
sqlStmt.append(") AS SELECT ");
if (cols == null || cols.length == 0) {
@@ -213,9 +194,7 @@ public abstract class NetezzaExternalTableImportMapper<K, V> extends
public void map(Integer dataSliceId, NullWritable val, Context context)
throws IOException, InterruptedException {
conf = context.getConfiguration();
- localLogDir =
- DirectNetezzaManager.getLocalLogDir(context.getTaskAttemptID());
- logDir = conf.get(DirectNetezzaManager.NETEZZA_LOG_DIR_OPT);
+
dbc = new DBConfiguration(conf);
numMappers = ConfigurationHelper.getConfNumMaps(conf);
@@ -250,9 +229,6 @@ public abstract class NetezzaExternalTableImportMapper<K, V> extends
extTableThread.printException();
throw new IOException(extTableThread.getException());
}
- FileUploader.uploadFilesToDFS(taskAttemptDir.getAbsolutePath(),
- localLogDir, logDir, context.getJobID().toString(),
- conf);
}
}
}