[kylin] branch master updated: KYLIN-3934 add config for sqoop config null-string and null-non-string

[ni...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/master by this push:
     new 3f0ca84  KYLIN-3934 add config for sqoop config null-string and null-non-string
3f0ca84 is described below

commit 3f0ca847e09532ff07d4258f92e30fec331e672c
Author: wanghao <wa...@yit.com>
AuthorDate: Tue Apr 9 10:11:47 2019 +0800

    KYLIN-3934 add config for sqoop config null-string and null-non-string
---
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java    | 8 ++++++++
 .../main/java/org/apache/kylin/source/jdbc/JdbcHiveInputBase.java | 8 +++++---
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ed7b1da..93dff1b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1050,6 +1050,14 @@ public abstract class KylinConfigBase implements Serializable {
         return Integer.parseInt(getOptional("kylin.source.jdbc.sqoop-mapper-num", "4"));
     }
 
+    public String getSqoopNullString() {
+        return getOptional("kylin.source.jdbc.sqoop-null-string", "\\\\N");
+    }
+
+    public String getSqoopNullNonString() {
+        return getOptional("kylin.source.jdbc.sqoop-null-non-string", "\\\\N");
+    }
+
     public Map<String, String> getSqoopConfigOverride() {
         return getPropertiesByPrefix("kylin.source.jdbc.sqoop-config-override.");
     }
diff --git a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveInputBase.java b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveInputBase.java
index 77e38c7..20f2dcb 100644
--- a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveInputBase.java
+++ b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveInputBase.java
@@ -170,6 +170,8 @@ public class JdbcHiveInputBase extends HiveInputBase {
             String jdbcUser = config.getJdbcSourceUser();
             String jdbcPass = config.getJdbcSourcePass();
             String sqoopHome = config.getSqoopHome();
+            String sqoopNullString = config.getSqoopNullString();
+            String sqoopNullNonString = config.getSqoopNullNonString();
             String filedDelimiter = config.getJdbcSourceFieldDelimiter();
             int mapperNum = config.getSqoopMapperNum();
 
@@ -196,10 +198,10 @@ public class JdbcHiveInputBase extends HiveInputBase {
             String cmd = String.format(Locale.ROOT,
                     "%s/bin/sqoop import" + generateSqoopConfigArgString()
                             + "--connect \"%s\" --driver %s --username %s --password \"%s\" --query \"%s AND \\$CONDITIONS\" "
-                            + "--target-dir %s/%s --split-by %s --boundary-query \"%s\" --null-string '' "
-                            + "--fields-terminated-by '%s' --num-mappers %d",
+                            + "--target-dir %s/%s --split-by %s --boundary-query \"%s\" --null-string '%s' "
+                            + "--null-non-string '%s' --fields-terminated-by '%s' --num-mappers %d",
                     sqoopHome, connectionUrl, driverClass, jdbcUser, jdbcPass, selectSql, jobWorkingDir, hiveTable,
-                    splitColumn, bquery, filedDelimiter, mapperNum);
+                    splitColumn, bquery, sqoopNullString, sqoopNullNonString, filedDelimiter, mapperNum);
             logger.debug(String.format(Locale.ROOT, "sqoop cmd:%s", cmd));
             CmdStep step = new CmdStep();
             step.setCmd(cmd);