You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/02/03 19:43:34 UTC
[27/50] [abbrv] carbondata git commit: [CARBONDATA-2120]Fixed is null
filter issue
[CARBONDATA-2120]Fixed is null filter issue
Problem: Is null filter is failing for numeric data type(No dictionary column).
Root cause: Min max calculation is wrong when no dictionary column is not the first column.
As it is not the first column null value can come in between and min max for null value is getting updated only when first row is null
Solution: Update the min max in all the case when value is null or not null for all type
This closes #1912
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6c097cbf
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6c097cbf
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6c097cbf
Branch: refs/heads/branch-1.3
Commit: 6c097cbf310e8d2199e57cde4fcc417122a8a1ca
Parents: 27ec651
Author: kumarvishal <ku...@gmail.com>
Authored: Fri Feb 2 17:53:57 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Feb 3 00:23:39 2018 +0530
----------------------------------------------------------------------
.../page/statistics/LVStringStatsCollector.java | 28 ++++++++---------
.../core/util/path/CarbonTablePath.java | 2 +-
.../src/test/resources/newsample.csv | 7 +++++
.../testsuite/filterexpr/TestIsNullFilter.scala | 32 ++++++++++++++++++++
4 files changed, 53 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c097cbf/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
index 61acec9..23795c5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/LVStringStatsCollector.java
@@ -73,28 +73,26 @@ public class LVStringStatsCollector implements ColumnPageStatsCollector {
@Override
public void update(byte[] value) {
// input value is LV encoded
+ byte[] newValue = null;
assert (value.length >= 2);
if (value.length == 2) {
assert (value[0] == 0 && value[1] == 0);
- if (min == null && max == null) {
- min = new byte[0];
- max = new byte[0];
- }
- return;
+ newValue = new byte[0];
+ } else {
+ int length = (value[0] << 8) + (value[1] & 0xff);
+ assert (length > 0);
+ newValue = new byte[value.length - 2];
+ System.arraycopy(value, 2, newValue, 0, newValue.length);
}
- int length = (value[0] << 8) + (value[1] & 0xff);
- assert (length > 0);
- byte[] v = new byte[value.length - 2];
- System.arraycopy(value, 2, v, 0, v.length);
if (min == null && max == null) {
- min = v;
- max = v;
+ min = newValue;
+ max = newValue;
} else {
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(min, v) > 0) {
- min = v;
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(min, newValue) > 0) {
+ min = newValue;
}
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(max, v) < 0) {
- max = v;
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(max, newValue) < 0) {
+ max = newValue;
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c097cbf/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index d8c64c4..5a63d2f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -75,7 +75,7 @@ public class CarbonTablePath extends Path {
* @param carbonFilePath
*/
public static String getFolderContainingFile(String carbonFilePath) {
- return carbonFilePath.substring(0, carbonFilePath.lastIndexOf(File.separator));
+ return carbonFilePath.substring(0, carbonFilePath.lastIndexOf('/'));
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c097cbf/integration/spark-common-test/src/test/resources/newsample.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/newsample.csv b/integration/spark-common-test/src/test/resources/newsample.csv
new file mode 100644
index 0000000..38cd3dd
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/newsample.csv
@@ -0,0 +1,7 @@
+id,name,time
+1,'one',2014-01-01 08:00:00
+2,'two',2014-01-02 08:02:00
+3,'three',2014-01-03 08:03:00
+4,'four',2014-01-04 08:04:00
+null,null,null
+5,'five',2020-10-20 08:01:20
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c097cbf/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestIsNullFilter.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestIsNullFilter.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestIsNullFilter.scala
new file mode 100644
index 0000000..9e85c9e
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestIsNullFilter.scala
@@ -0,0 +1,32 @@
+package org.apache.carbondata.spark.testsuite.filterexpr
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class TestIsNullFilter extends QueryTest with BeforeAndAfterAll {
+ override def beforeAll: Unit = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ sql("drop table if exists main")
+ sql("create table main(id int, name string, time timestamp) STORED BY 'org.apache.carbondata.format'")
+ sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/newsample.csv' into table main OPTIONS('bad_records_action'='force')")
+ }
+
+ test("select * from main where time is null") {
+ checkAnswer(
+ sql("select count(*) from main where time is null"),
+ Seq(Row(1)))
+ }
+
+ override def afterAll: Unit = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ sql("drop table if exists main")
+ }
+
+}