You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2013/03/22 04:55:14 UTC
[1/2] git commit: SQOOP-932: Netezza direct manager is ignoring
--null-*-string arguments
Updated Branches:
refs/heads/trunk e93ec9f39 -> 7048ca302
SQOOP-932: Netezza direct manager is ignoring --null-*-string arguments
(Venkat Ranganathan via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/a0a03d7f
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/a0a03d7f
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/a0a03d7f
Branch: refs/heads/trunk
Commit: a0a03d7f313c11472ebc6c6acd2d97cde36f8417
Parents: e93ec9f
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Thu Mar 21 20:23:31 2013 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Thu Mar 21 20:23:31 2013 -0700
----------------------------------------------------------------------
src/java/org/apache/sqoop/lib/DelimiterSet.java | 14 ++++
.../apache/sqoop/manager/DirectNetezzaManager.java | 40 +++++++++-
.../netezza/NetezzaExternalTableExportMapper.java | 16 +++-
.../netezza/NetezzaExternalTableImportMapper.java | 11 +++-
.../netezza/NetezzaExternalTableExportJob.java | 42 +++++++----
.../netezza/NetezzaExternalTableImportJob.java | 47 ++++++++-----
.../manager/DirectNetezzaExportManualTest.java | 56 +++++++++++++-
.../sqoop/manager/NetezzaExportManualTest.java | 13 ++++
.../sqoop/manager/NetezzaImportManualTest.java | 10 ++-
9 files changed, 201 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0a03d7f/src/java/org/apache/sqoop/lib/DelimiterSet.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/lib/DelimiterSet.java b/src/java/org/apache/sqoop/lib/DelimiterSet.java
index ef62ba0..d76890e 100644
--- a/src/java/org/apache/sqoop/lib/DelimiterSet.java
+++ b/src/java/org/apache/sqoop/lib/DelimiterSet.java
@@ -46,6 +46,20 @@ public class DelimiterSet implements Cloneable {
"sqoop.output.escaped.by";
public static final String OUTPUT_ENCLOSE_REQUIRED_KEY =
"sqoop.output.enclose.required";
+
+ /**
+ * Create the input variations for export instead of overloading them.
+ */
+ public static final String INPUT_FIELD_DELIM_KEY =
+ "sqoop.input.field.delim";
+ public static final String INPUT_RECORD_DELIM_KEY =
+ "sqoop.input.record.delim";
+ public static final String INPUT_ENCLOSED_BY_KEY =
+ "sqoop.input.enclosed.by";
+ public static final String INPUT_ESCAPED_BY_KEY =
+ "sqoop.input.escaped.by";
+ public static final String INPUT_ENCLOSE_REQUIRED_KEY =
+ "sqoop.input.enclose.required";
/**
* Create a delimiter set with the default delimiters
* (comma for fields, newline for records).
http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0a03d7f/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
index ef98936..4f36bf6 100644
--- a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
+++ b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
@@ -28,6 +28,8 @@ import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.ParseException;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -59,13 +61,39 @@ public class DirectNetezzaManager extends NetezzaManager {
private static final String QUERY_CHECK_DICTIONARY_FOR_TABLE =
"SELECT 1 FROM _V_OBJECTS WHERE OWNER= ? "
+ " AND OBJNAME = ? and OBJTYPE = 'TABLE'";
-
+ public static final String NETEZZA_NULL_VALUE =
+ "netezza.exttable.null.value";
public DirectNetezzaManager(SqoopOptions opts) {
super(opts);
try {
handleNetezzaExtraArgs(options);
- } catch (ParseException ioe) {
- throw new RuntimeException(ioe.getMessage(), ioe);
+ } catch (ParseException pe) {
+ throw new RuntimeException(pe.getMessage(), pe);
+ }
+ }
+
+ private void checkNullValueStrings(String nullStrValue,
+ String nullNonStrValue) throws IOException {
+
+ if (!StringUtils.equals(nullStrValue, nullNonStrValue)) {
+ throw new IOException(
+ "Detected different values of --input-string and --input-non-string "
+ + "parameters. Netezza direct manager does not support that. Please "
+ + "either use the same values or omit the --direct parameter.");
+ }
+
+
+ // Null String values cannot be more 4 chars in length in the case
+ // Netezza external tables.
+
+ if (nullStrValue != null) {
+ nullStrValue = StringEscapeUtils.unescapeJava(nullStrValue);
+ if (nullStrValue.length() > 4) {
+ throw new IOException(
+ "Null string (and null non string) values for Netezza direct mode"
+ + " manager must be less than 4 characters in length");
+ }
+ options.getConf().set(NETEZZA_NULL_VALUE, nullStrValue);
}
}
@@ -131,12 +159,13 @@ public class DirectNetezzaManager extends NetezzaManager {
options = context.getOptions();
context.setConnManager(this);
-
checkTable(); // Throws excpetion as necessary
NetezzaExternalTableExportJob exporter = null;
char qc = (char) options.getInputEnclosedBy();
char ec = (char) options.getInputEscapedBy();
+ checkNullValueStrings(options.getInNullStringValue(),
+ options.getInNullNonStringValue());
if (qc > 0 && !(qc == '"' || qc == '\'')) {
throw new ExportException("Input enclosed-by character must be '\"' "
@@ -173,6 +202,9 @@ public class DirectNetezzaManager extends NetezzaManager {
new IOException("Null tableName for Netezza external table import.");
}
+ checkNullValueStrings(options.getNullStringValue(),
+ options.getNullNonStringValue());
+
char qc = options.getOutputEnclosedBy();
char ec = options.getOutputEscapedBy();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0a03d7f/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableExportMapper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableExportMapper.java b/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableExportMapper.java
index 410a569..22b7af5 100644
--- a/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableExportMapper.java
+++ b/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableExportMapper.java
@@ -64,9 +64,11 @@ public abstract class NetezzaExternalTableExportMapper<K, V> extends
private String getSqlStatement() throws IOException {
- char fd = (char) conf.getInt(DelimiterSet.OUTPUT_FIELD_DELIM_KEY, ',');
- char qc = (char) conf.getInt(DelimiterSet.OUTPUT_ENCLOSED_BY_KEY, 0);
- char ec = (char) conf.getInt(DelimiterSet.OUTPUT_ESCAPED_BY_KEY, 0);
+ char fd = (char) conf.getInt(DelimiterSet.INPUT_FIELD_DELIM_KEY, ',');
+ char qc = (char) conf.getInt(DelimiterSet.INPUT_ENCLOSED_BY_KEY, 0);
+ char ec = (char) conf.getInt(DelimiterSet.INPUT_ESCAPED_BY_KEY, 0);
+
+ String nullValue = conf.get(DirectNetezzaManager.NETEZZA_NULL_VALUE);
int errorThreshold = conf.getInt(
DirectNetezzaManager.NETEZZA_ERROR_THRESHOLD_OPT, 1);
@@ -89,7 +91,13 @@ public abstract class NetezzaExternalTableExportMapper<K, V> extends
}
sqlStmt.append(" FORMAT 'Text' ");
sqlStmt.append(" INCLUDEZEROSECONDS TRUE ");
- sqlStmt.append(" NULLVALUE 'NULL' ");
+ sqlStmt.append(" NULLVALUE '");
+ if (nullValue != null) {
+ sqlStmt.append(nullValue);
+ } else {
+ sqlStmt.append("null");
+ }
+ sqlStmt.append("' ");
if (qc > 0) {
switch (qc) {
case '\'':
http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0a03d7f/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java b/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java
index 9e6cab6..bcdc9e1 100644
--- a/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java
+++ b/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java
@@ -66,6 +66,9 @@ public class NetezzaExternalTableImportMapper extends
char fd = (char) conf.getInt(DelimiterSet.OUTPUT_FIELD_DELIM_KEY, ',');
char qc = (char) conf.getInt(DelimiterSet.OUTPUT_ENCLOSED_BY_KEY, 0);
char ec = (char) conf.getInt(DelimiterSet.OUTPUT_ESCAPED_BY_KEY, 0);
+
+ String nullValue = conf.get(DirectNetezzaManager.NETEZZA_NULL_VALUE);
+
int errorThreshold = conf.getInt(
DirectNetezzaManager.NETEZZA_ERROR_THRESHOLD_OPT, 1);
String logDir = conf.get(DirectNetezzaManager.NETEZZA_LOG_DIR_OPT);
@@ -86,7 +89,13 @@ public class NetezzaExternalTableImportMapper extends
}
sqlStmt.append(" FORMAT 'Text' ");
sqlStmt.append(" INCLUDEZEROSECONDS TRUE ");
- sqlStmt.append(" NULLVALUE 'null' ");
+ sqlStmt.append(" NULLVALUE '");
+ if (nullValue != null) {
+ sqlStmt.append(nullValue);
+ } else {
+ sqlStmt.append("null");
+ }
+ sqlStmt.append("' ");
if (qc > 0) {
switch (qc) {
case '\'':
http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0a03d7f/src/java/org/apache/sqoop/mapreduce/netezza/NetezzaExternalTableExportJob.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/netezza/NetezzaExternalTableExportJob.java b/src/java/org/apache/sqoop/mapreduce/netezza/NetezzaExternalTableExportJob.java
index 2a702d9..7caf9be 100644
--- a/src/java/org/apache/sqoop/mapreduce/netezza/NetezzaExternalTableExportJob.java
+++ b/src/java/org/apache/sqoop/mapreduce/netezza/NetezzaExternalTableExportJob.java
@@ -20,6 +20,7 @@ package org.apache.sqoop.mapreduce.netezza;
import java.io.IOException;
+import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -28,6 +29,7 @@ import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.sqoop.lib.DelimiterSet;
import org.apache.sqoop.manager.ConnManager;
+import org.apache.sqoop.manager.DirectNetezzaManager;
import org.apache.sqoop.mapreduce.DBWritable;
import
org.apache.sqoop.mapreduce.db.netezza.NetezzaExternalTableRecordExportMapper;
@@ -52,29 +54,41 @@ public class NetezzaExternalTableExportJob extends ExportJobBase {
}
@Override
+ protected void propagateOptionsToJob(Job job) {
+ Configuration conf = job.getConfiguration();
+ String nullValue = options.getInNullStringValue();
+ if (nullValue != null) {
+ conf.set(DirectNetezzaManager.NETEZZA_NULL_VALUE,
+ StringEscapeUtils.unescapeJava(nullValue));
+ }
+ conf.setInt(DelimiterSet.INPUT_FIELD_DELIM_KEY,
+ options.getInputFieldDelim());
+ conf.setInt(DelimiterSet.INPUT_RECORD_DELIM_KEY,
+ options.getInputRecordDelim());
+ conf.setInt(DelimiterSet.INPUT_ENCLOSED_BY_KEY,
+ options.getInputEnclosedBy());
+ // Netezza uses \ as the escape character. Force the use of it
+ int escapeChar = options.getInputEscapedBy();
+ if (escapeChar > 0) {
+ if (escapeChar != '\\') {
+ LOG.info(
+ "Setting escaped char to \\ for Netezza external table export");
+ }
+ conf.setInt(DelimiterSet.INPUT_ESCAPED_BY_KEY, '\\');
+ }
+ conf.setBoolean(DelimiterSet.INPUT_ENCLOSE_REQUIRED_KEY,
+ options.isOutputEncloseRequired());
+ }
/**
* Configure the inputformat to use for the job.
*/
+ @Override
protected void configureInputFormat(Job job, String tableName,
String tableClassName, String splitByCol) throws ClassNotFoundException,
IOException {
// Configure the delimiters, etc.
Configuration conf = job.getConfiguration();
- conf.setInt(DelimiterSet.OUTPUT_FIELD_DELIM_KEY,
- options.getInputFieldDelim());
- conf.setInt(DelimiterSet.OUTPUT_RECORD_DELIM_KEY,
- options.getInputRecordDelim());
- conf.setInt(DelimiterSet.OUTPUT_ENCLOSED_BY_KEY,
- options.getInputEnclosedBy());
- // Netezza uses \ as the escape character. Force the use of it
- int escapeChar = options.getOutputEscapedBy();
- if (escapeChar > 0 && escapeChar != '\\') {
- LOG.info("Setting escaped char to \\ for Netezza external table import");
- conf.setInt(DelimiterSet.OUTPUT_ESCAPED_BY_KEY, '\\');
- }
- conf.setBoolean(DelimiterSet.OUTPUT_ENCLOSE_REQUIRED_KEY,
- options.isOutputEncloseRequired());
ConnManager mgr = context.getConnManager();
String username = options.getUsername();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0a03d7f/src/java/org/apache/sqoop/mapreduce/netezza/NetezzaExternalTableImportJob.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/netezza/NetezzaExternalTableImportJob.java b/src/java/org/apache/sqoop/mapreduce/netezza/NetezzaExternalTableImportJob.java
index 7ee6f70..0f7c1b0 100644
--- a/src/java/org/apache/sqoop/mapreduce/netezza/NetezzaExternalTableImportJob.java
+++ b/src/java/org/apache/sqoop/mapreduce/netezza/NetezzaExternalTableImportJob.java
@@ -20,10 +20,12 @@ package org.apache.sqoop.mapreduce.netezza;
import java.io.IOException;
+import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.sqoop.lib.DelimiterSet;
+import org.apache.sqoop.manager.DirectNetezzaManager;
import org.apache.sqoop.mapreduce.DBWritable;
import org.apache.sqoop.mapreduce.ImportJobBase;
import org.apache.sqoop.mapreduce.RawKeyTextOutputFormat;
@@ -47,10 +49,37 @@ public class NetezzaExternalTableImportJob extends ImportJobBase {
context);
}
+ @Override
+ protected void propagateOptionsToJob(Job job) {
+ Configuration conf = job.getConfiguration();
+ String nullValue = options.getNullStringValue();
+ if (nullValue != null) {
+ conf.set(DirectNetezzaManager.NETEZZA_NULL_VALUE,
+ StringEscapeUtils.unescapeJava(nullValue));
+ }
+ conf.setInt(DelimiterSet.OUTPUT_FIELD_DELIM_KEY,
+ options.getOutputFieldDelim());
+ conf.setInt(DelimiterSet.OUTPUT_RECORD_DELIM_KEY,
+ options.getOutputRecordDelim());
+ conf.setInt(DelimiterSet.OUTPUT_ENCLOSED_BY_KEY,
+ options.getOutputEnclosedBy());
+ // Netezza uses \ as the escape character. Force the use of it
+ int escapeChar = options.getOutputEscapedBy();
+ if (escapeChar > 0) {
+ if (escapeChar != '\\') {
+ LOG.info(
+ "Setting escaped char to \\ for Netezza external table import");
+ }
+ conf.setInt(DelimiterSet.OUTPUT_ESCAPED_BY_KEY, '\\');
+ }
+ conf.setBoolean(DelimiterSet.OUTPUT_ENCLOSE_REQUIRED_KEY,
+ options.isOutputEncloseRequired());
+
+ }
/**
* Configure the inputformat to use for the job.
*/
-
+ @Override
protected void configureInputFormat(Job job, String tableName,
String tableClassName, String splitByCol) throws ClassNotFoundException,
IOException {
@@ -89,22 +118,6 @@ public class NetezzaExternalTableImportJob extends ImportJobBase {
DataDrivenDBInputFormat.setInput(job, DBWritable.class, tableName,
whereClause, mgr.escapeColName(splitByCol), sqlColNames);
- Configuration conf = job.getConfiguration();
- conf.setInt(DelimiterSet.OUTPUT_FIELD_DELIM_KEY,
- options.getOutputFieldDelim());
- conf.setInt(DelimiterSet.OUTPUT_RECORD_DELIM_KEY,
- options.getOutputRecordDelim());
- conf.setInt(DelimiterSet.OUTPUT_ENCLOSED_BY_KEY,
- options.getOutputEnclosedBy());
- // Netezza uses \ as the escape character. Force the use of it
- int escapeChar = options.getOutputEscapedBy();
- if (escapeChar > 0 && escapeChar != '\\') {
- LOG.info("Setting escaped char to \\ for Netezza external table import");
- conf.setInt(DelimiterSet.OUTPUT_ESCAPED_BY_KEY, '\\');
- }
- conf.setBoolean(DelimiterSet.OUTPUT_ENCLOSE_REQUIRED_KEY,
- options.isOutputEncloseRequired());
-
LOG.debug("Using InputFormat: " + inputFormatClass);
job.setInputFormatClass(getInputFormatClass());
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0a03d7f/src/test/com/cloudera/sqoop/manager/DirectNetezzaExportManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/DirectNetezzaExportManualTest.java b/src/test/com/cloudera/sqoop/manager/DirectNetezzaExportManualTest.java
index 938ffc5..aace924 100644
--- a/src/test/com/cloudera/sqoop/manager/DirectNetezzaExportManualTest.java
+++ b/src/test/com/cloudera/sqoop/manager/DirectNetezzaExportManualTest.java
@@ -25,13 +25,9 @@ import java.util.Arrays;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sqoop.manager.DirectNetezzaManager;
-import org.junit.After;
-import org.junit.Before;
import org.junit.Test;
-
import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.TestExport.ColumnGenerator;
/**
* Test the DirectNetezzaManager implementation's exportJob() functionality.
@@ -162,7 +158,59 @@ public class DirectNetezzaExportManualTest extends NetezzaExportManualTest {
runNetezzaTest(getTableName(), argv);
}
+ @Test
+ public void testNullStringExport() throws Exception {
+
+ String [] extraArgs = {
+ "--input-null-string", "\\\\N",
+ "--input-null-non-string", "\\\\N",
+ "--input-escaped-by", "\\",
+ };
+ ColumnGenerator[] extraCols = new ColumnGenerator[] {
+ new NullColumnGenerator(),
+ };
+ String[] argv = getArgv(true, 10, 10, extraArgs);
+ runNetezzaTest(getTableName(), argv, extraCols);
+ }
+
+
+ public void testDifferentNullStrings() throws IOException, SQLException {
+ ColumnGenerator[] extraCols = new ColumnGenerator[] {
+ new NullColumnGenerator(),
+ };
+
+ String [] extraArgs = {
+ "--input-null-string", "\\N",
+ "--input-null-non-string", "\\M",
+ };
+ String[] argv = getArgv(true, 10, 10, extraArgs);
+ try {
+ runNetezzaTest(getTableName(), argv, extraCols);
+ fail("Expected failure for different null strings");
+ } catch(IOException ioe) {
+ // success
+ }
+ }
+
+ @Test(expected = java.io.IOException.class)
+ public void testLongNullStrings() throws IOException, SQLException {
+ ColumnGenerator[] extraCols = new ColumnGenerator[] {
+ new NullColumnGenerator(),
+ };
+
+ String [] extraArgs = {
+ "--input-null-string", "morethan4chars",
+ "--input-null-non-string", "morethan4chars",
+ };
+ String[] argv = getArgv(true, 10, 10, extraArgs);
+ try {
+ runNetezzaTest(getTableName(), argv, extraCols);
+ fail("Expected failure for long null strings");
+ } catch(IOException ioe) {
+ // success
+ }
+ }
@Override
public void testMultiMapTextExportWithStaging() throws IOException,
http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0a03d7f/src/test/com/cloudera/sqoop/manager/NetezzaExportManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/NetezzaExportManualTest.java b/src/test/com/cloudera/sqoop/manager/NetezzaExportManualTest.java
index 50d27fe..43dd254 100644
--- a/src/test/com/cloudera/sqoop/manager/NetezzaExportManualTest.java
+++ b/src/test/com/cloudera/sqoop/manager/NetezzaExportManualTest.java
@@ -243,4 +243,17 @@ public class NetezzaExportManualTest extends TestExport {
w.close();
os.close();
}
+
+ protected class NullColumnGenerator implements ColumnGenerator {
+ public String getExportText(int rowNum) {
+ return "\\N";
+ }
+ public String getVerifyText(int rowNum) {
+ return null;
+ }
+
+ public String getType() {
+ return "INTEGER";
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0a03d7f/src/test/com/cloudera/sqoop/manager/NetezzaImportManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/NetezzaImportManualTest.java b/src/test/com/cloudera/sqoop/manager/NetezzaImportManualTest.java
index 3482dd8..86f5bdd 100644
--- a/src/test/com/cloudera/sqoop/manager/NetezzaImportManualTest.java
+++ b/src/test/com/cloudera/sqoop/manager/NetezzaImportManualTest.java
@@ -205,13 +205,14 @@ public class NetezzaImportManualTest extends ImportJobTestCase {
private String[] getDirectModeExpectedResultsWithNulls() {
String [] expectedResults = {
- "1,Aaron,2009-05-14,1000000,T,engineering,nvl,1",
- "2,Bob,2009-04-20,400,T,sales,nvl,2",
- "3,Fred,2009-01-23,15,F,marketing,nvl,3",
+ "1,Aaron,2009-05-14,1000000,T,engineering,\\N,1",
+ "2,Bob,2009-04-20,400,T,sales,\\N,2",
+ "3,Fred,2009-01-23,15,F,marketing,\\N,3",
};
return expectedResults;
}
+
private String[] getArgv(boolean isDirect, String tableName,
String... extraArgs) {
ArrayList<String> args = new ArrayList<String>();
@@ -325,7 +326,7 @@ public class NetezzaImportManualTest extends ImportJobTestCase {
}
@Test
- public void testNullEscapeCharacters() throws Exception {
+ public void testNullStringValue() throws Exception {
String [] extraArgs = {
@@ -355,4 +356,5 @@ public class NetezzaImportManualTest extends ImportJobTestCase {
runNetezzaTest(true, tableName, expectedResults,
extraArgs);
}
+
}
[2/2] git commit: SQOOP-962: Document null string behavior
Posted by ja...@apache.org.
SQOOP-962: Document null string behavior
(Venkat Ranganathan via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/7048ca30
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/7048ca30
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/7048ca30
Branch: refs/heads/trunk
Commit: 7048ca30236b8f6a24c147d27a168c0ef2d150cb
Parents: a0a03d7
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Thu Mar 21 20:50:26 2013 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Thu Mar 21 20:50:26 2013 -0700
----------------------------------------------------------------------
src/docs/user/connectors.txt | 56 +++++++++++++++++++++++++++++++++++++
1 files changed, 56 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/7048ca30/src/docs/user/connectors.txt
----------------------------------------------------------------------
diff --git a/src/docs/user/connectors.txt b/src/docs/user/connectors.txt
index c172c4b..cf66112 100644
--- a/src/docs/user/connectors.txt
+++ b/src/docs/user/connectors.txt
@@ -310,3 +310,59 @@ $ sqoop export \
--export-dir hdfsdir \
--input-fields-terminated-by "\t"
----
+
+Null string handling
+^^^^^^^^^^^^^^^^^^^^
+
+Netezza direct connector supports the null-string features of Sqoop. The null
+string values are converted to appropriate external table options during export
+and import operations.
+
+.Supported export control arguments:
+[grid="all"]
+`----------------------------------------`---------------------------------------
+Argument Description
+---------------------------------------------------------------------------------
++\--input-null-string <null-string>+ The string to be interpreted as\
+ null for string columns.
++\--input-null-non-string <null-string>+ The string to be interpreted as\
+ null for non string columns.
+---------------------------------------------------------------------------------
+
+In the case of Netezza direct mode connector, both the arguments must be
+left to the default values or explicitly set to the same value. Furthermore
+the null string value is restricted to 0-4 utf8 characters.
+
+On export, for non-string columns, if the chosen null value is a valid
+representation in the column domain, then the column might not be loaded as
+null. For example, if the null string value is specified as "1", then on
+export, any occurrence of "1" in the input file will be loaded as value 1
+instead of NULL for int columns.
+
+It is suggested that the null value be specified as empty string for
+performance and consistency.
+
+.Supported import control arguments:
+[grid="all"]
+`----------------------------------------`---------------------------------------
+Argument Description
+---------------------------------------------------------------------------------
++\--null-string <null-string>+ The string to be interpreted as\
+ null for string columns.
++\--null-non-string <null-string>+ The string to be interpreted as\
+ null for non string columns.
+---------------------------------------------------------------------------------
+
+In the case of Netezza direct mode connector, both the arguments must be
+left to the default values or explicitly set to the same value. Furthermore
+the null string value is restricted to 0-4 utf8 characters.
+
+On import, for non-string columns, the chosen null value in current
+implementations the null value representation is ignored for non character
+columns. For example, if the null string value is specified as "\N", then on
+import, any occurrence of NULL for non-char columns in the table will be
+imported as an empty string instead of '\N', the chosen null string
+representation.
+
+It is suggested that the null value be specified as empty string for
+performance and consistency.