[carbondata] branch master updated: [CARBONDATA-3854] Quotechar support to more than one character

[aj...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

ajantha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new daee357  [CARBONDATA-3854] Quotechar support to more than one character
daee357 is described below

commit daee357673a2dc194c2b1d5762baee40f47d23f3
Author: Mahesh Raju Somalaraju <ma...@huawei.com>
AuthorDate: Thu Jun 11 22:33:18 2020 +0530

    [CARBONDATA-3854] Quotechar support to more than one character
    
    Why is this PR needed?
    Need to support more than one character the same as a like delimiter.
    Quote char support to unprintable character like \u0009 \u0010
    Currently, carbondata will not support setting quotechar to printable char like \u0009.
    the current behaviour is quotechar will through exception if we give more than one character.
    
    What changes were proposed in this PR?
    support quote char with more than 1 length and use unescape char while
    setting.
    
    Does this PR introduce any user interface change?
    No
    
    Is any new testcase added?
    No
    
    This closes #3790
---
 .../scala/org/apache/spark/sql/catalyst/CarbonParserUtil.scala    | 8 --------
 .../carbondata/spark/testsuite/dataload/TestLoadOptions.scala     | 8 --------
 .../processing/loading/model/CarbonLoadModelBuilder.java          | 3 ++-
 3 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/catalyst/CarbonParserUtil.scala b/integration/spark/src/main/scala/org/apache/spark/sql/catalyst/CarbonParserUtil.scala
index 135a143..7619afd 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/catalyst/CarbonParserUtil.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/catalyst/CarbonParserUtil.scala
@@ -946,14 +946,6 @@ object CarbonParserUtil {
       throw new MalformedCarbonCommandException(errorMessage)
     }
 
-    // Validate QUOTECHAR length
-    if (options.exists(_._1.equalsIgnoreCase("QUOTECHAR"))) {
-      val quoteChar: String = options.get("quotechar").get.head._2
-      if (quoteChar.length > 1 ) {
-        throw new MalformedCarbonCommandException("QUOTECHAR cannot be more than one character.")
-      }
-    }
-
     // Validate COMMENTCHAR length
     if (options.exists(_._1.equalsIgnoreCase("COMMENTCHAR"))) {
       val commentChar: String = options.get("commentchar").get.head._2
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadOptions.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadOptions.scala
index 88d7829..d4ae6ac 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadOptions.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadOptions.scala
@@ -35,14 +35,6 @@ class TestLoadOptions extends QueryTest with BeforeAndAfterAll {
     sql("drop table if exists TestLoadTableOptions")
   }
 
-  test("test load data with more than one char in quotechar option") {
-    val errorMessage = intercept[MalformedCarbonCommandException] {
-      sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/dataretention1.csv' INTO TABLE " +
-          s"TestLoadTableOptions OPTIONS('QUOTECHAR'='\\\\')")
-    }.getMessage
-    assert(errorMessage.equals("QUOTECHAR cannot be more than one character."))
-  }
-
   test("test load data with more than one char in commentchar option") {
     val errorMessage = intercept[MalformedCarbonCommandException] {
       sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/dataretention1.csv' INTO TABLE " +
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
index 33eefa5..d2bb47e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
@@ -181,7 +181,8 @@ public class CarbonLoadModelBuilder {
 
     validateGlobalSortPartitions(global_sort_partitions);
     carbonLoadModel.setEscapeChar(checkDefaultValue(optionsFinal.get("escapechar"), "\\"));
-    carbonLoadModel.setQuoteChar(checkDefaultValue(optionsFinal.get("quotechar"), "\""));
+    carbonLoadModel.setQuoteChar(
+        CarbonUtil.unescapeChar(checkDefaultValue(optionsFinal.get("quotechar"), "\"")));
     carbonLoadModel.setCommentChar(checkDefaultValue(optionsFinal.get("commentchar"), "#"));
     String lineSeparator = CarbonUtil.unescapeChar(options.get("line_separator"));
     if (lineSeparator != null) {