[carbondata] branch master updated: [CARBONDATA-3341] fixed invalid NULL result in filter query

[ku...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

kumarvishal09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new a6ab97c  [CARBONDATA-3341] fixed invalid NULL result in filter query
a6ab97c is described below

commit a6ab97ca40427af5225f12a063a0e44221a503e1
Author: kunal642 <ku...@gmail.com>
AuthorDate: Thu Apr 4 11:53:05 2019 +0530

    [CARBONDATA-3341] fixed invalid NULL result in filter query
    
    Problem: When vector filter push down is true and the table contains a null value
    then thegetNullBitSet method is giving an byte[]to represent null.
    But there is no check for the value of the bitset.
    
    Solution: Check if null bit set length is 0 then set the same to the chunkData.
    
    This closes #3172
---
 .../core/datastore/chunk/store/ColumnPageWrapper.java      |  7 ++++++-
 .../spark/testsuite/sortcolumns/TestSortColumns.scala      | 14 ++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
index a1c4aec..f4d3fe4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
@@ -261,7 +261,12 @@ public class ColumnPageWrapper implements DimensionColumnPage {
       // if the compare value is null and the data is also null we can directly return 0
       return 0;
     } else {
-      byte[] chunkData = this.getChunkDataInBytes(rowId);
+      byte[] chunkData;
+      if (nullBitSet != null && nullBitSet.length == 0) {
+        chunkData = nullBitSet;
+      } else {
+        chunkData = this.getChunkDataInBytes(rowId);
+      }
       return ByteUtil.UnsafeComparer.INSTANCE.compareTo(chunkData, compareValue);
     }
   }
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index df97d0f..bbd58c0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -385,6 +385,17 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
         "sort_columns is unsupported for double datatype column: empno"))
   }
 
+  test("test if equal to 0 filter on sort column gives correct result") {
+    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_PUSH_ROW_FILTERS_FOR_VECTOR,
+      "true")
+    sql("create table test1(a bigint) stored by 'carbondata' TBLPROPERTIES('sort_columns'='a')")
+    sql("insert into test1 select 'k'")
+    sql("insert into test1 select '1'")
+    assert(sql("select * from test1 where a = 1 or a = 0").count() == 1)
+    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_PUSH_ROW_FILTERS_FOR_VECTOR,
+      CarbonCommonConstants.CARBON_PUSH_ROW_FILTERS_FOR_VECTOR_DEFAULT)
+  }
+
   override def afterAll = {
     dropTestTables
     CarbonProperties.getInstance().addProperty(
@@ -392,9 +403,12 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
     CarbonProperties.getInstance()
       .addProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
         CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT)
+    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_PUSH_ROW_FILTERS_FOR_VECTOR,
+      CarbonCommonConstants.CARBON_PUSH_ROW_FILTERS_FOR_VECTOR_DEFAULT)
   }
 
   def dropTestTables = {
+    sql("drop table if exists test1")
     sql("drop table if exists sortint")
     sql("drop table if exists sortint1")
     sql("drop table if exists sortlong")