You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/11/18 15:29:36 UTC
[02/28] carbondata git commit: [CARBONDATA-1720][FILTER] Wrong data
displayed for <= filter for timestamp column(dictionary column)
[CARBONDATA-1720][FILTER] Wrong data displayed for <= filter for timestamp column(dictionary column)
Issue:
<= filter is giving wrong results for timestamp dictioinary column
Solution:
In less than equal to filter, we are considering surrogate 2 as default value. But surrogate 1 is for default value.
This closes #1502
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/17892b17
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/17892b17
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/17892b17
Branch: refs/heads/fgdatamap
Commit: 17892b17b688eaa637b3dd97c25286edb4183eaa
Parents: 09d0205
Author: dhatchayani <dh...@gmail.com>
Authored: Wed Nov 15 18:41:00 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Nov 16 16:12:19 2017 +0530
----------------------------------------------------------------------
.../executer/RangeValueFilterExecuterImpl.java | 2 +-
...velRangeLessThanEqualFilterExecuterImpl.java | 10 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 14 +-
.../src/test/resources/timestamp.csv | 301 +++++++++++++++++++
.../RangeFilterAllDataTypesTestCases.scala | 9 +
5 files changed, 327 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index 0cfa198..ee373c5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -554,7 +554,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
if (dimColEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(dimColEvaluatorInfo.getDimension().getDataType());
- int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1;
+ int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
CarbonDimension currentBlockDimension =
segmentProperties.getDimensions().get(dimensionBlocksIndex);
if (currentBlockDimension.isSortColumn()) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 5e0bb41..88cf75c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -267,7 +267,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(
dimColEvaluatorInfoList.get(0).getDimension().getDataType());
- int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1;
+ int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
CarbonDimension currentBlockDimension =
segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
if (currentBlockDimension.isSortColumn()) {
@@ -324,7 +324,9 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
return bitSet;
}
} else {
- skip = start;
+ // as start will be last index of null value inclusive
+ // so adding 1 to skip last null value
+ skip = start + 1;
}
startIndex = skip;
}
@@ -392,7 +394,9 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
return bitSet;
}
} else {
- skip = start;
+ // as start will be last index of null value inclusive
+ // so adding 1 to skip last null value
+ skip = start + 1;
}
startIndex = skip;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index 74162f6..7f735c2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -269,7 +269,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(
dimColEvaluatorInfoList.get(0).getDimension().getDataType());
- int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1;
+ int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
CarbonDimension currentBlockDimension =
segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
if (currentBlockDimension.isSortColumn()) {
@@ -319,7 +319,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
if (null != defaultValue) {
start = CarbonUtil
.getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
- defaultValue, false);
+ defaultValue, true);
if (start < 0) {
skip = -(start + 1);
// end of block
@@ -327,7 +327,9 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
return bitSet;
}
} else {
- skip = start;
+ // as start will be last index of null value inclusive
+ // so adding 1 to skip last null value
+ skip = start + 1;
}
startIndex = skip;
}
@@ -395,7 +397,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
if (null != defaultValue) {
start = CarbonUtil
.getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex,
- numerOfRows - 1, defaultValue, false);
+ numerOfRows - 1, defaultValue, true);
if (start < 0) {
skip = -(start + 1);
// end of block
@@ -403,7 +405,9 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
return bitSet;
}
} else {
- skip = start;
+ // as start will be last index of null value inclusive
+ // so adding 1 to skip last null value
+ skip = start + 1;
}
startIndex = skip;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/integration/spark-common-test/src/test/resources/timestamp.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/timestamp.csv b/integration/spark-common-test/src/test/resources/timestamp.csv
new file mode 100644
index 0000000..0a7b3ff
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/timestamp.csv
@@ -0,0 +1,301 @@
+timestampCol
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
index 00be125..87cc194 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
@@ -263,6 +263,15 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
)
}
+ test("test range filter for less than filter"){
+ sql("drop table if exists timestampTable")
+ sql("create table timestampTable (timestampCol timestamp) stored by 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='timestampCol')")
+ sql(s"load data local inpath '$resourcesPath/timestamp.csv' into table timestampTable")
+ checkAnswer(sql("select * from timestampTable where timestampCol='1970-01-01 05:30:00'"),
+ sql("select * from timestampTable where timestampCol<='1970-01-01 05:30:00'"))
+ sql("drop table if exists timestampTable")
+ }
+
test("test range filter for direct dictionary not equality"){
checkAnswer(
sql("select doj from directDictionaryTable where doj != '2016-03-14 15:00:16'"),