You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/03/15 09:57:28 UTC
[1/2] incubator-carbondata git commit: Fixed filter not null issue
Repository: incubator-carbondata
Updated Branches:
refs/heads/master dca475ba7 -> 0ba07adaa
Fixed filter not null issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f1f45810
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f1f45810
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f1f45810
Branch: refs/heads/master
Commit: f1f458100f4f1ec0d5a29bac4cdc7dfcce1298a3
Parents: dca475b
Author: kumarvishal <ku...@gmail.com>
Authored: Tue Mar 14 17:02:56 2017 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Mar 15 15:26:36 2017 +0530
----------------------------------------------------------------------
.../visitor/DictionaryColumnVisitor.java | 12 +++-
.../carbondata/core/util/DataTypeUtil.java | 17 +++++-
.../src/test/resources/filter/notNullFilter.csv | 11 ++++
.../filterexpr/TestNotNullFilter.scala | 60 ++++++++++++++++++++
4 files changed, 95 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f1f45810/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
index bdd2dda..b4812be 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
@@ -37,7 +37,7 @@ public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf {
* @param visitableObj
* @param metadata
* @throws FilterUnsupportedException,if exception occurs while evaluating
- * filter models.
+ * filter models.
* @throws IOException
* @throws FilterUnsupportedException
*/
@@ -56,8 +56,14 @@ public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf {
if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
// Adding default surrogate key of null member inorder to not display the same while
// displaying the report as per hive compatibility.
- resolvedFilterObject.getFilterList()
- .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ // first check of surrogate key for null value is already added then
+ // no need to add again otherwise result will be wrong in case of exclude filter
+ // this is because two times it will flip the same bit
+ if (resolvedFilterObject.getFilterList()
+ .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ }
Collections.sort(resolvedFilterObject.getFilterList());
}
visitableObj.setFilterValues(resolvedFilterObject);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f1f45810/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 5f69753..1d983e0 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -507,12 +507,25 @@ public final class DataTypeUtil {
switch (dimension.getDataType()) {
case DECIMAL:
return parseStringToBigDecimal(value, dimension);
+ case INT:
+ Integer.parseInt(value);
+ break;
+ case DOUBLE:
+ Double.parseDouble(value);
+ break;
+ case LONG:
+ Long.parseLong(value);
+ break;
+ case FLOAT:
+ Float.parseFloat(value);
+ break;
default:
- return value;
+ // do nothing
}
- } catch (Exception e) {
+ } catch (NumberFormatException e) {
return null;
}
+ return value;
}
private static String parseStringToBigDecimal(String value, CarbonDimension dimension) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f1f45810/integration/spark-common-test/src/test/resources/filter/notNullFilter.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/filter/notNullFilter.csv b/integration/spark-common-test/src/test/resources/filter/notNullFilter.csv
new file mode 100644
index 0000000..3e8c268
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/filter/notNullFilter.csv
@@ -0,0 +1,11 @@
+ID,date,country,name,phonetype,serialname,salary,floatField
+1,2015/7/23,china,aaa1,phone197,ASD69643,15000,2.34
+vishal,2015/7/24,china,aaa2,phone756,ASD42892,15001,2.34
+raghu,2015/7/25,china,aaa3,phone1904,ASD37014,15002,2.34
+4,2015/7/26,china,aaa4,phone2435,ASD66902,15003,2.34
+5,2015/7/27,china,aaa5,phone2441,ASD90633,15004,2.34
+6,2015/7/28,china,aaa6,phone294,ASD59961,15005,3.5
+7,2015/7/29,china,aaa7,phone610,ASD14875,15006,2.34
+8,2015/7/30,china,aaa8,phone1848,ASD57308,15007,2.34
+9,2015/7/18,china,aaa9,phone706,ASD86717,15008,2.34
+10,2015/7/19,usa,aaa10,phone685,ASD30505,15009,2.34
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f1f45810/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestNotNullFilter.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestNotNullFilter.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestNotNullFilter.scala
new file mode 100644
index 0000000..8a532ac
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestNotNullFilter.scala
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.filterexpr
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+/**
+ * Test cases for testing columns having \N or \null values for non numeric columns
+ */
+class TestNotNullFilter extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ sql("drop table if exists carbonTableNotNull")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+ val csvFilePath = s"$resourcesPath/filter/notNullFilter.csv"
+ sql("""
+ CREATE TABLE IF NOT EXISTS carbonTableNotNull
+ (ID Int, date timestamp, country String,
+ name String, phonetype String, serialname char(10), salary Int)
+ STORED BY 'carbondata' TBLPROPERTIES ('DICTIONARY_INCLUDE'='ID')
+ """)
+ sql(s"""
+ LOAD DATA LOCAL INPATH '$csvFilePath' into table carbonTableNotNull OPTIONS('BAD_RECORDS_ACTION'='FORCE')
+ """)
+ }
+
+
+ test("select ID from carbonTableNotNull where ID is not null") {
+ checkAnswer(
+ sql("select ID from carbonTableNotNull where ID is not null"),
+ Seq(Row(1), Row(4), Row(5), Row(6), Row(7), Row(8), Row(9), Row(10)))
+ }
+
+ override def afterAll {
+ sql("drop table if exists carbonTableNotNull")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ }
+}
[2/2] incubator-carbondata git commit: [CARBONDATA-770] Fixed Not
null filter query data mismatch issue. This closes #654
Posted by ra...@apache.org.
[CARBONDATA-770] Fixed Not null filter query data mismatch issue. This closes #654
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/0ba07ada
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/0ba07ada
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/0ba07ada
Branch: refs/heads/master
Commit: 0ba07adaaaa558388c2254643a995e9bbe438c0c
Parents: dca475b f1f4581
Author: ravipesala <ra...@gmail.com>
Authored: Wed Mar 15 15:27:14 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Mar 15 15:27:14 2017 +0530
----------------------------------------------------------------------
.../visitor/DictionaryColumnVisitor.java | 12 +++-
.../carbondata/core/util/DataTypeUtil.java | 17 +++++-
.../src/test/resources/filter/notNullFilter.csv | 11 ++++
.../filterexpr/TestNotNullFilter.scala | 60 ++++++++++++++++++++
4 files changed, 95 insertions(+), 5 deletions(-)
----------------------------------------------------------------------