You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/09/10 15:38:39 UTC
carbondata git commit: [CARBONDATA-1379] Fixed Date range filter with
cast not working
Repository: carbondata
Updated Branches:
refs/heads/master 252c3e335 -> 4030cfb27
[CARBONDATA-1379] Fixed Date range filter with cast not working
This closes #1254
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4030cfb2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4030cfb2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4030cfb2
Branch: refs/heads/master
Commit: 4030cfb27795e7d8dea6dadd7573bc0e3265a437
Parents: 252c3e3
Author: Ravindra Pesala <ra...@gmail.com>
Authored: Sat Aug 12 11:42:26 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sun Sep 10 23:38:21 2017 +0800
----------------------------------------------------------------------
.../timestamp/DateDirectDictionaryGenerator.java | 13 +++----------
.../core/scan/expression/ExpressionResult.java | 8 +++++++-
.../DateDataTypeDirectDictionaryTest.scala | 11 +++++++++++
3 files changed, 21 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4030cfb2/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
index 0d7cb6c..5a6e03d 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
@@ -18,7 +18,6 @@ package org.apache.carbondata.core.keygenerator.directdictionary.timestamp;
import java.text.ParseException;
import java.text.SimpleDateFormat;
-import java.util.Calendar;
import java.util.Date;
import java.util.TimeZone;
@@ -37,16 +36,10 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
private static final int cutOffDate = Integer.MAX_VALUE >> 1;
private static final long SECONDS_PER_DAY = 60 * 60 * 24L;
- private static final long MILLIS_PER_DAY = SECONDS_PER_DAY * 1000L;
+ public static final long MILLIS_PER_DAY = SECONDS_PER_DAY * 1000L;
private ThreadLocal<SimpleDateFormat> simpleDateFormatLocal = new ThreadLocal<>();
- //Java TimeZone has no mention of thread safety. Use thread local instance to be safe.
- private ThreadLocal<TimeZone> threadLocalLocalTimeZone = new ThreadLocal() {
- @Override protected TimeZone initialValue() {
- return Calendar.getInstance().getTimeZone();
- }
- };
private String dateFormat;
/**
@@ -154,14 +147,14 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
}
private int generateKey(long timeValue) {
- long milli = timeValue + threadLocalLocalTimeZone.get().getOffset(timeValue);
- return (int) Math.floor((double) milli / MILLIS_PER_DAY) + cutOffDate;
+ return (int) Math.floor((double) timeValue / MILLIS_PER_DAY) + cutOffDate;
}
public void initialize() {
if (simpleDateFormatLocal.get() == null) {
simpleDateFormatLocal.set(new SimpleDateFormat(dateFormat));
simpleDateFormatLocal.get().setLenient(false);
+ simpleDateFormatLocal.get().setTimeZone(TimeZone.getTimeZone("GMT"));
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4030cfb2/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
index 74e666b..08b1972 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
@@ -24,8 +24,10 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import java.util.TimeZone;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.DateDirectDictionaryGenerator;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -177,6 +179,9 @@ public class ExpressionResult implements Comparable<ExpressionResult> {
case TIMESTAMP:
String format = CarbonUtil.getFormatFromProperty(this.getDataType());
SimpleDateFormat parser = new SimpleDateFormat(format);
+ if (this.getDataType() == DataType.DATE) {
+ parser.setTimeZone(TimeZone.getTimeZone("GMT"));
+ }
if (value instanceof Timestamp) {
return parser.format((Timestamp) value);
} else if (value instanceof java.sql.Date) {
@@ -187,7 +192,8 @@ public class ExpressionResult implements Comparable<ExpressionResult> {
}
return parser.format(new Timestamp((long) value));
} else if (value instanceof Integer) {
- return parser.format(new java.sql.Date((long)value));
+ long date = ((int) value) * DateDirectDictionaryGenerator.MILLIS_PER_DAY;
+ return parser.format(new java.sql.Date(date));
}
return value.toString();
default:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4030cfb2/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala
index 9018ec0..697b495 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala
@@ -122,6 +122,17 @@ class DateDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfterAll
)
}
+ test("select doj from directDictionaryTable with greater than filter with cast") {
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj > date('2016-03-14')"),
+ Seq(Row(Date.valueOf("2016-04-14")))
+ )
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj > cast('2016-03-14' as date)"),
+ Seq(Row(Date.valueOf("2016-04-14")))
+ )
+ }
+
test("select count(doj) from directDictionaryTable") {
checkAnswer(
sql("select count(doj) from directDictionaryTable"),