[carbondata] branch master updated: [CARBONDATA-4018]Fix CSV header validation not contains dimension columns

[in...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

indhumuthumurugesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new f05af2e  [CARBONDATA-4018]Fix CSV header validation not contains dimension columns
f05af2e is described below

commit f05af2ecc478e4fdbc4fda710d7a11448c5fc2c7
Author: akashrn5 <ak...@gmail.com>
AuthorDate: Mon Sep 28 20:59:25 2020 +0530

    [CARBONDATA-4018]Fix CSV header validation not contains dimension columns
    
    Why is this PR needed?
    CSV header validation not considering the dimension columns in schema
    
    What changes were proposed in this PR?
    refactor the code to consider the dimensions in schema for csv header validation
    
    Does this PR introduce any user interface change?
    No
    
    Is any new testcase added?
    No
    
    This closes #3963
---
 .../dataload/TestLoadDataWithFileHeaderException.scala       | 12 ++++++++++--
 .../carbondata/processing/util/CarbonDataProcessorUtil.java  |  6 +++---
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala
index 547c201..e6809bf 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala
@@ -91,14 +91,22 @@ class TestLoadDataWithFileHeaderException extends QueryTest with BeforeAndAfterA
   }
 
   test("test load data with header=false and wrong fileheader") {
-    val e = intercept[Exception] {
+    val e1 = intercept[Exception] {
       sql(
         s"""
         LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3
         options('header'='false', 'fileheader'='ID1,date2,country,name,phonetype,serialname,salary')
         """)
     }
-    assert(e.getMessage.contains("CSV header in DDL is not proper. Column names in schema and CSV header are not the same"))
+    val e2 = intercept[Exception] {
+      sql(
+        s"""
+        LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3
+        options('header'='false', 'fileheader'='ID,date2,country,name,phonetype,serialnames,salary')
+        """)
+    }
+    assert(e1.getMessage.contains("CSV header in DDL is not proper. Column names in schema and CSV header are not the same"))
+    assert(e2.getMessage.contains("CSV header in DDL is not proper. Column names in schema and CSV header are not the same"))
   }
 
   test("test load data with header=true, but without fileheader") {
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 474a54e..1037455 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -336,10 +336,10 @@ public final class CarbonDataProcessorUtil {
     Map<String, String> properties =
         schema.getCarbonTable().getTableInfo().getFactTable().getTableProperties();
     String spatialProperty = properties.get(CarbonCommonConstants.SPATIAL_INDEX);
+    spatialProperty = spatialProperty != null ? spatialProperty.trim() : null;
     for (CarbonDimension dimension : dimensions) {
-      if (spatialProperty != null && !dimension.getColName()
-          .equalsIgnoreCase(spatialProperty.trim())) {
-        // skip the non-schema column
+      // skip the non-schema column
+      if (!dimension.getColName().equalsIgnoreCase(spatialProperty)) {
         columnNames.add(dimension.getColName());
       }
     }