You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/08/01 10:05:23 UTC
[25/47] incubator-carbondata git commit: Problem: Null values getting
display in not equals filter for timestamp datatype column (#878)
Problem: Null values getting display in not equals filter for timestamp datatype column (#878)
Analysis: While setting surrogates for exclude filter in custom dictionary visitor, surrogate for null value is not getting added to surrogate list due to which the query displays null value in case of dictionary column filters.
Impact: Not equals, equals, not null and is null query on timestamp datatype column
Fix: Add surrogate value for null member in exclude filter list
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e367e12b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e367e12b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e367e12b
Branch: refs/heads/master
Commit: e367e12b1c2523f12887af03d2a87547d4e897ae
Parents: 82b7407
Author: manishgupta88 <to...@gmail.com>
Authored: Wed Jul 27 20:30:49 2016 +0530
Committer: sujith71955 <su...@gmail.com>
Committed: Wed Jul 27 20:30:49 2016 +0530
----------------------------------------------------------------------
.../visitor/CustomTypeDictionaryVisitor.java | 8 ++++++
.../visitor/DictionaryColumnVisitor.java | 4 ++-
.../spark/src/test/resources/datasample.csv | 1 +
.../dataload/TestLoadDataWithNoMeasure.scala | 2 +-
...estampDataTypeDirectDictionaryTestCase.scala | 29 ++++++++++++++++++--
...TypeDirectDictionaryWithNoDictTestCase.scala | 3 +-
6 files changed, 42 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
index fa0f0e9..a2e5e21 100644
--- a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
+++ b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
@@ -64,6 +64,14 @@ public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorInt
resolvedFilterObject = getDirectDictionaryValKeyMemberForFilter(metadata.getTableIdentifier(),
metadata.getColumnExpression(), evaluateResultListFinal, metadata.isIncludeFilter(),
isNotTimestampType);
+ if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject
+ .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
+ // Adding default surrogate key of null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ Collections.sort(resolvedFilterObject.getFilterList());
+ }
visitableObj.setFilterValues(resolvedFilterObject);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
index 333a254..8488ad9 100644
--- a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
+++ b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.query.carbon.executor.exception.QueryExecutionException;
import org.carbondata.query.expression.exception.FilterIllegalMemberException;
import org.carbondata.query.expression.exception.FilterUnsupportedException;
@@ -61,7 +62,8 @@ public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf {
if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
// Adding default surrogate key of null member inorder to not display the same while
// displaying the report as per hive compatibility.
- resolvedFilterObject.getFilterList().add(1);
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
Collections.sort(resolvedFilterObject.getFilterList());
}
} catch (QueryExecutionException e) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/integration/spark/src/test/resources/datasample.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/datasample.csv b/integration/spark/src/test/resources/datasample.csv
index fdf80e6..2f8ef35 100644
--- a/integration/spark/src/test/resources/datasample.csv
+++ b/integration/spark/src/test/resources/datasample.csv
@@ -1,3 +1,4 @@
empno,doj,salary
11,2016-04-14 15:00:09,5040.56
12,2016-03-14 15:00:09,1040.56
+13,,1040.56
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
index f88e74c..9aa4189 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
@@ -48,7 +48,7 @@ class TestLoadDataWithNoMeasure extends QueryTest with BeforeAndAfterAll {
checkAnswer(
sql("select empno from nomeasureTest"),
- Seq(Row("11"), Row("12"))
+ Seq(Row("11"), Row("12"), Row("13"))
)
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
index baff810..f325254 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
@@ -70,15 +70,40 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
}
}
- test("select doj from directDictionaryCube") {
+ test("test direct dictionary for not null condition") {
checkAnswer(
- sql("select doj from directDictionaryCube"),
+ sql("select doj from directDictionaryCube where doj is not null"),
Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
Row(Timestamp.valueOf("2016-04-14 15:00:09.0"))
)
)
}
+ test("test direct dictionary for getting all the values") {
+ checkAnswer(
+ sql("select doj from directDictionaryCube"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
+ Row(Timestamp.valueOf("2016-04-14 15:00:09.0")),
+ Row(null)
+ )
+ )
+ }
+
+ test("test direct dictionary for not equals condition") {
+ checkAnswer(
+ sql("select doj from directDictionaryCube where doj != '2016-04-14 15:00:09.0'"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0"))
+ )
+ )
+ }
+
+ test("test direct dictionary for null condition") {
+ checkAnswer(
+ sql("select doj from directDictionaryCube where doj is null"),
+ Seq(Row(null)
+ )
+ )
+ }
test("select doj from directDictionaryCube with equals filter") {
checkAnswer(
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
index a62c632..c654e95 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
@@ -74,7 +74,8 @@ class TimestampDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with
checkAnswer(
sql("select doj from directDictionaryTable"),
Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
- Row(Timestamp.valueOf("2016-04-14 15:00:09.0"))
+ Row(Timestamp.valueOf("2016-04-14 15:00:09.0")),
+ Row(null)
)
)
}