You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/06/08 11:57:15 UTC
incubator-kylin git commit: KYLIN-817 Support Extract() on timestamp
column
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8.0 cb10c562f -> 1b0237219
KYLIN-817 Support Extract() on timestamp column
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1b023721
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1b023721
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1b023721
Branch: refs/heads/0.8.0
Commit: 1b0237219bc5b0c3aa21209a016bb63bdb2f9b1a
Parents: cb10c56
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Jun 8 16:57:08 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Jun 8 17:56:35 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/DateFormat.java | 2 +-
.../apache/kylin/dict/DictionaryGenerator.java | 13 ++++++-----
.../kylin/query/relnode/ColumnRowType.java | 6 ++---
.../kylin/query/relnode/OLAPProjectRel.java | 24 +++++++++++++-------
.../org/apache/kylin/storage/tuple/Tuple.java | 4 +---
5 files changed, 27 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1b023721/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
index c6a651f..f74debd 100644
--- a/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
+++ b/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -70,7 +70,7 @@ public class DateFormat {
return stringToDate(str, DEFAULT_DATE_PATTERN).getTime();
} else if (str.length() == 19) {
return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS).getTime();
- } else if (str.length() == 23) {
+ } else if (str.length() > 19) {
return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).getTime();
} else {
throw new IllegalArgumentException("there is no valid date pattern for:" + str);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1b023721/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index a8e27b3..c132d29 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -54,7 +54,10 @@ public class DictionaryGenerator {
// build dict, case by data type
if (dataType.isDateTimeFamily()) {
- dict = buildDateTimeDict(values, baseId, nSamples, samples);
+ if (dataType.isDate())
+ dict = buildDateDict(values, baseId, nSamples, samples);
+ else
+ dict = new TimeStrDictionary(); // base ID is always 0
} else if (dataType.isNumberFamily()) {
dict = buildNumberDict(values, baseId, nSamples, samples);
} else {
@@ -113,7 +116,7 @@ public class DictionaryGenerator {
return dict;
}
- private static Dictionary buildDateTimeDict(Collection<byte[]> values, int baseId, int nSamples, ArrayList samples) {
+ private static Dictionary buildDateDict(Collection<byte[]> values, int baseId, int nSamples, ArrayList samples) {
final int BAD_THRESHOLD = 0;
String matchPattern = null;
@@ -131,7 +134,7 @@ public class DictionaryGenerator {
if (samples.size() < nSamples && samples.contains(str) == false)
samples.add(str);
} catch (ParseException e) {
- logger.info("Unrecognized datetime value: " + str);
+ logger.info("Unrecognized date value: " + str);
badCount++;
if (badCount > BAD_THRESHOLD) {
matchPattern = null;
@@ -144,9 +147,7 @@ public class DictionaryGenerator {
}
}
- //FIXME: except for date type, all other date time family types are treated as TimeStrDictionary
- logger.info("Using TimeStrDictionary");
- return new TimeStrDictionary();
+ throw new IllegalStateException("Unrecognized datetime value");
}
private static Dictionary buildStringDict(Collection<byte[]> values, int baseId, int nSamples, ArrayList samples) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1b023721/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java b/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java
index 176d9e4..095b5e2 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java
@@ -32,10 +32,8 @@ import org.apache.kylin.metadata.model.TblColRef;
public class ColumnRowType {
private List<TblColRef> columns;
- // for calculated column, like (CASE LSTG_FORMAT_NAME WHEN 'Auction' THEN
- // '111' ELSE '222' END)
- // source columns are the contributing physical columns, here the
- // LSTG_FORMAT_NAME
+ // for calculated column, like (CASE LSTG_FORMAT_NAME WHEN 'Auction' THEN '111' ELSE '222' END)
+ // source columns are the contributing physical columns, here the LSTG_FORMAT_NAME
private List<Set<TblColRef>> sourceColumns;
public ColumnRowType(List<TblColRef> columns) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1b023721/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
index 8de3101..6a15c4f 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
@@ -148,6 +148,20 @@ public class OLAPProjectRel extends ProjectRelBase implements OLAPRel, Enumerabl
return column;
}
+ private TblColRef translateFirstRexInputRef(RexCall call, ColumnRowType inputColumnRowType, String fieldName, Set<TblColRef> sourceCollector) {
+ for (RexNode operand : call.getOperands()) {
+ if (operand instanceof RexInputRef) {
+ return translateRexInputRef((RexInputRef) operand, inputColumnRowType, fieldName, sourceCollector);
+ }
+ if (operand instanceof RexCall) {
+ TblColRef r = translateFirstRexInputRef((RexCall) operand, inputColumnRowType, fieldName, sourceCollector);
+ if (r != null)
+ return r;
+ }
+ }
+ return null;
+ }
+
private TblColRef translateRexInputRef(RexInputRef inputRef, ColumnRowType inputColumnRowType, String fieldName, Set<TblColRef> sourceCollector) {
int index = inputRef.getIndex();
// check it for rewrite count
@@ -175,16 +189,10 @@ public class OLAPProjectRel extends ProjectRelBase implements OLAPRel, Enumerabl
private TblColRef translateRexCall(RexCall call, ColumnRowType inputColumnRowType, String fieldName, Set<TblColRef> sourceCollector) {
SqlOperator operator = call.getOperator();
if (operator == SqlStdOperatorTable.EXTRACT_DATE) {
- List<RexNode> extractDateOps = call.getOperands();
- RexCall reinterpret = (RexCall) extractDateOps.get(1);
- List<RexNode> reinterpretOps = reinterpret.getOperands();
- RexInputRef inputRef = (RexInputRef) reinterpretOps.get(0);
- return translateRexInputRef(inputRef, inputColumnRowType, fieldName, sourceCollector);
+ return translateFirstRexInputRef(call, inputColumnRowType, fieldName, sourceCollector);
} else if (operator instanceof SqlUserDefinedFunction) {
if (operator.getName().equals("QUARTER")) {
- List<RexNode> quaterOps = call.getOperands();
- RexInputRef inputRef = (RexInputRef) quaterOps.get(0);
- return translateRexInputRef(inputRef, inputColumnRowType, fieldName, sourceCollector);
+ return translateFirstRexInputRef(call, inputColumnRowType, fieldName, sourceCollector);
}
} else if (operator instanceof SqlCaseOperator) {
for (RexNode operand : call.getOperands()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1b023721/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
index 950068b..8a85c3a 100644
--- a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
+++ b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
@@ -181,7 +181,7 @@ public class Tuple implements ITuple {
// convert epoch time
return dateToEpicDays(strValue);// Optiq expects Integer instead of Long. by honma
} else if ("timestamp".equals(dataTypeName) || "datetime".equals(dataTypeName)) {
- return Long.parseLong(strValue);
+ return Long.valueOf(DateFormat.stringToMillis(strValue));
} else if ("tinyint".equals(dataTypeName)) {
return Byte.valueOf(strValue);
} else if ("short".equals(dataTypeName) || "smallint".equals(dataTypeName)) {
@@ -194,8 +194,6 @@ public class Tuple implements ITuple {
return Double.valueOf(strValue);
} else if ("decimal".equals(dataTypeName)) {
return new BigDecimal(strValue);
- } else if ("timestamp".equals(dataTypeName)) {
- return Long.valueOf(DateFormat.stringToMillis(strValue));
} else if ("float".equals(dataTypeName)){
return Float.valueOf(strValue);
} else {