[carbondata] branch master updated: [CARBONDATA-4214] inserting NULL value when timestamp value received from FROM_UNIXTIME(0)

[in...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

indhumuthumurugesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new 18665cc  [CARBONDATA-4214] inserting NULL value when timestamp value received from FROM_UNIXTIME(0)
18665cc is described below

commit 18665cc0b72d455ff66bfa63501faec94025b492
Author: Mahesh Raju Somalaraju <ma...@huawei.com>
AuthorDate: Fri Jun 18 00:40:18 2021 +0530

    [CARBONDATA-4214] inserting NULL value when timestamp value received from FROM_UNIXTIME(0)
    
    Why is this PR needed?
    Filling null in case of timestamp value is received from FROM_UNIXTIME(0) as spark original
    insert rdd value[internalRow] received in this case zero. if the original column
    value[internalRow] is zero then in insert flow adding NULL and giving NULL to spark.
    When query happens on the same column received NULL value instead of timestamp value.
    Problem code: if (internalRow.getLong(index) == 0) { internalRow.setNullAt(index) }
    
    What changes were proposed in this PR?
    Removed the null filling check for zero value case and if internalRow value is non
    null/empty then only set the internalRow timestamp value.
    
    Does this PR introduce any user interface change?
    No
    
    Is any new testcase added?
    Yes
    
    This closes #4154
---
 .../command/management/CommonLoadUtils.scala       |  5 ++--
 .../allqueries/AllDataTypesTestCase.scala          | 29 ++++++++++++++++++++++
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
index 695681a..f2a8280 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
@@ -781,9 +781,8 @@ object CommonLoadUtils {
         internalRowOriginal
       }
       for (index <- timeStampIndex) {
-        if (internalRow.getLong(index) == 0) {
-          internalRow.setNullAt(index)
-        } else {
+        // timestmap value can be set other than null/empty case
+        if (!internalRow.isNullAt(index)) {
           internalRow.setLong(
             index,
             internalRow.getLong(index) / TimeStampGranularityTypeValue.MILLIS_SECONDS.getValue)
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase.scala
index 7044729..b95755c 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase.scala
@@ -17,6 +17,8 @@
 
 package org.apache.carbondata.spark.testsuite.allqueries
 
+import java.sql.Timestamp
+
 import org.apache.spark.sql.{Row, SaveMode}
 import org.apache.spark.sql.test.util.QueryTest
 import org.scalatest.BeforeAndAfterAll
@@ -73,6 +75,33 @@ class AllDataTypesTestCase extends QueryTest with BeforeAndAfterAll {
     }
   }
 
+  test("insert data with from_unixtime(0) and query") {
+    sql("drop table if exists time_carbon1")
+    sql("drop table if exists time_parquet")
+    sql("create table if not exists time_carbon1(time1 timestamp) stored as carbondata")
+    sql("create table if not exists time_parquet(time1 timestamp) stored as parquet")
+    sql("insert into time_carbon1 select from_unixtime(0)")
+    sql("insert into time_parquet select from_unixtime(0)")
+    sql("insert into time_carbon1 select from_unixtime(123456)")
+    sql("insert into time_parquet select from_unixtime(123456)")
+    checkAnswer(sql("select * from time_carbon1"),
+      sql("select * from time_parquet"))
+    sql("drop table if exists time_carbon1")
+    sql("drop table if exists time_parquet")
+  }
+
+  test("insert data with empty/null and query") {
+    sql("drop table if exists time_carbon2")
+    sql("create table if not exists time_carbon2(time1 timestamp) stored as carbondata")
+    sql("insert into time_carbon2 select null")
+    checkAnswer(sql("select count(*) from time_carbon2"), Seq(Row(1)))
+    checkAnswer(sql("select * from time_carbon2"), Seq(Row(null)))
+    sql("insert into time_carbon2 select ''")
+    checkAnswer(sql("select count(*) from time_carbon2"), Seq(Row(2)))
+    checkAnswer(sql("select * from time_carbon2"), Seq(Row(null), Row(null)))
+    sql("drop table if exists time_carbon2")
+  }
+
   // Test-24
   test("select channelsId, sum(channelsId+ 10)  Total from Carbon_automation_test group by  channelsId order by Total") {
     checkAnswer(