You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by qi...@apache.org on 2020/10/12 12:00:45 UTC

[carbondata] branch master updated: [CARBONDATA-4008] Fixed IN filter on date column is returning 0 results when 'carbon.push.rowfilters.for.vector' is true

This is an automated email from the ASF dual-hosted git repository.

qiangcai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new b4d96f4  [CARBONDATA-4008] Fixed IN filter on date column is returning 0 results when 'carbon.push.rowfilters.for.vector' is true
b4d96f4 is described below

commit b4d96f4be959d2a965e80213cc970f5e5c293e7b
Author: Venu Reddy <k....@gmail.com>
AuthorDate: Thu Sep 24 01:31:58 2020 +0530

    [CARBONDATA-4008] Fixed IN filter on date column is returning 0 results when 'carbon.push.rowfilters.for.vector' is true
    
    Why is this PR needed?
    IN filter on date column is returning 0 results when 'carbon.push.rowfilters.for.vector' is set to true. RowLevelFilterExecutorImpl.applyFilter() calls createRow() and applies filter at row level with expression.evaluate(row) invocation. expression in this case is InExpression. But createRow() missed to fill the date column value. Thus expression always evaluates to false and returns 0 rows.
    
    What changes were proposed in this PR?
    Filled the date column value in createRow().
    Have removed unused imports introduced in previous PR.(SqlAstBuilderHelper.scala)
    
    Does this PR introduce any user interface change?
    No
    
    Is any new testcase added?
    Yes
    
    This closes #3953
---
 .../executer/RowLevelFilterExecutorImpl.java       | 17 ++++++++--
 .../spark/sql/hive/SqlAstBuilderHelper.scala       |  5 ++-
 .../spark/testsuite/filterexpr/TestInFilter.scala  | 38 +++++++++++++++++-----
 3 files changed, 46 insertions(+), 14 deletions(-)

diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecutorImpl.java
index a5b3a42..de31fdd 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecutorImpl.java
@@ -36,6 +36,8 @@ import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
 import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
 import org.apache.carbondata.core.datastore.chunk.store.ColumnPageWrapper;
 import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
 import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.DateDirectDictionaryGenerator;
 import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityTypeValue;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -106,6 +108,11 @@ public class RowLevelFilterExecutorImpl implements FilterExecutor {
    */
   boolean isNaturalSorted;
 
+  /**
+   * date direct dictionary generator
+   */
+  private DirectDictionaryGenerator dateDictionaryGenerator;
+
   public RowLevelFilterExecutorImpl(List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
       List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
       AbsoluteTableIdentifier tableIdentifier, SegmentProperties segmentProperties,
@@ -138,6 +145,8 @@ public class RowLevelFilterExecutorImpl implements FilterExecutor {
     this.exp = exp;
     this.tableIdentifier = tableIdentifier;
     this.complexDimensionInfoMap = complexDimensionInfoMap;
+    this.dateDictionaryGenerator =
+        DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataTypes.DATE);
     initDimensionChunkIndexes();
     initMeasureChunkIndexes();
   }
@@ -438,9 +447,11 @@ public class RowLevelFilterExecutorImpl implements FilterExecutor {
         DimensionColumnPage columnDataChunk =
             blockChunkHolder.getDimensionRawColumnChunks()[dimensionChunkIndex[i]]
                 .decodeColumnPage(pageIndex);
-        if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataTypes.DATE &&
-            (columnDataChunk instanceof VariableLengthDimensionColumnPage ||
-                columnDataChunk instanceof ColumnPageWrapper)) {
+        if (dimColumnEvaluatorInfo.getDimension().getDataType() == DataTypes.DATE) {
+          record[dimColumnEvaluatorInfo.getRowIndex()] = dateDictionaryGenerator
+              .getValueFromSurrogate(ByteUtil.toInt(columnDataChunk.getChunkData(index), 0));
+        } else if (columnDataChunk instanceof VariableLengthDimensionColumnPage
+            || columnDataChunk instanceof ColumnPageWrapper) {
 
           byte[] memberBytes = columnDataChunk.getChunkData(index);
           if (null != memberBytes) {
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/SqlAstBuilderHelper.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/SqlAstBuilderHelper.scala
index 2ffb007..be949b4 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/SqlAstBuilderHelper.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/SqlAstBuilderHelper.scala
@@ -18,14 +18,13 @@
 package org.apache.spark.sql.hive
 
 import org.apache.spark.sql.catalyst.CarbonParserUtil
-import org.apache.spark.sql.catalyst.parser.ParserUtils.{string, withOrigin}
 import org.apache.spark.sql.catalyst.parser.SqlBaseParser
-import org.apache.spark.sql.catalyst.parser.SqlBaseParser.{AddTableColumnsContext, ChangeColumnContext, CreateTableContext, ShowTablesContext}
+import org.apache.spark.sql.catalyst.parser.SqlBaseParser.{AddTableColumnsContext, ChangeColumnContext, CreateTableContext}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.execution.SparkSqlAstBuilder
 import org.apache.spark.sql.execution.command.{AlterTableAddColumnsModel, AlterTableDataTypeChangeModel}
 import org.apache.spark.sql.execution.command.schema.{CarbonAlterTableAddColumnCommand, CarbonAlterTableColRenameDataTypeChangeCommand}
-import org.apache.spark.sql.execution.command.table.{CarbonExplainCommand, CarbonShowTablesCommand}
+import org.apache.spark.sql.execution.command.table.CarbonExplainCommand
 import org.apache.spark.sql.parser.CarbonSpark2SqlParser
 import org.apache.spark.sql.types.DecimalType
 
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestInFilter.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestInFilter.scala
index de3131e..27fc58f 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestInFilter.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestInFilter.scala
@@ -17,26 +17,29 @@
 
 package org.apache.carbondata.spark.testsuite.filterexpr
 
+import java.sql.{Date, Timestamp}
+
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
 
-class TestInFilter extends QueryTest with BeforeAndAfterAll{
+class TestInFilter extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach {
 
   override def beforeAll: Unit = {
     sql("drop table if exists test_table")
-    sql("create table test_table(intField INT, floatField FLOAT, doubleField DOUBLE, " +
-        "decimalField DECIMAL(18,2))  STORED AS carbondata")
-
     // turn on  row level filter in carbon
     // because only row level is on, 'in' will be pushdowned into CarbonScanRDD
     //  or in filter will be handled by spark.
     sql("set carbon.push.rowfilters.for.vector=true")
-    sql("insert into test_table values(8,8,8,8),(5,5.0,5.0,5.0),(4,1.00,2.00,3.00)," +
-        "(6,6.0000,6.0000,6.0000),(4743,4743.00,4743.0000,4743.0),(null,null,null,null)")
   }
 
   test("sql with in different measurement type") {
+    sql("create table test_table(intField INT, floatField FLOAT, doubleField DOUBLE, " +
+        "decimalField DECIMAL(18,2))  STORED AS carbondata")
+
+    sql("insert into test_table values(8,8,8,8),(5,5.0,5.0,5.0),(4,1.00,2.00,3.00)," +
+        "(6,6.0000,6.0000,6.0000),(4743,4743.00,4743.0000,4743.0),(null,null,null,null)")
+
     // the precision of filter value is less one digit than column value
     // float type test
     checkAnswer(
@@ -165,8 +168,27 @@ class TestInFilter extends QueryTest with BeforeAndAfterAll{
       Seq(Row(4, 1.00, 2.00, 3.00)))
   }
 
-  override def afterAll(): Unit = {
+  test("test infilter with date, timestamp columns") {
+    sql("create table test_table(i int, dt date, ts timestamp) stored as carbondata")
+    sql("insert into test_table select 1, '2020-03-30', '2020-03-30 10:00:00'")
+    sql("insert into test_table select 2, '2020-07-04', '2020-07-04 14:12:15'")
+    sql("insert into test_table select 3, '2020-09-23', '2020-09-23 12:30:45'")
+
+    checkAnswer(sql("select * from test_table where dt IN ('2020-03-30', '2020-09-23')"),
+      Seq(Row(1, Date.valueOf("2020-03-30"), Timestamp.valueOf("2020-03-30 10:00:00")),
+        Row(3, Date.valueOf("2020-09-23"), Timestamp.valueOf("2020-09-23 12:30:45"))))
+
+    checkAnswer(sql(
+      "select * from test_table where ts IN ('2020-03-30 10:00:00', '2020-07-04 14:12:15')"),
+      Seq(Row(1, Date.valueOf("2020-03-30"), Timestamp.valueOf("2020-03-30 10:00:00")),
+        Row(2, Date.valueOf("2020-07-04"), Timestamp.valueOf("2020-07-04 14:12:15"))))
+  }
+
+  override def afterEach(): Unit = {
     sql("drop table if exists test_table")
+  }
+
+  override def afterAll(): Unit = {
     sql("set carbon.push.rowfilters.for.vector=false")
     defaultConfig()
   }