You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/06/09 09:01:24 UTC
incubator-kylin git commit: KYLIN-817 fix bugs
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8.0 4b435b370 -> 4219fdac1
KYLIN-817 fix bugs
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4219fdac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4219fdac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4219fdac
Branch: refs/heads/0.8.0
Commit: 4219fdac1105e7736e8d51e1ab73da275b308b79
Parents: 4b435b3
Author: honma <ho...@ebay.com>
Authored: Tue Jun 9 15:00:35 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Jun 9 15:00:35 2015 +0800
----------------------------------------------------------------------
.../metadata/filter/DateConditionModifier.java | 32 ++++++++++++++------
.../apache/kylin/metadata/model/DataType.java | 17 ++++++++---
.../apache/kylin/rest/service/CacheService.java | 2 +-
.../kylin/streaming/KafkaClusterConfig.java | 1 -
4 files changed, 36 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4219fdac/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
index 8d5c10c..08be3de 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
@@ -3,6 +3,7 @@ package org.apache.kylin.metadata.filter;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
import java.util.Collection;
@@ -11,21 +12,17 @@ import java.util.List;
import java.util.Set;
/**
+ * Calcite passed down all time family constants as GregorianCalendar,
+ * we'll have to reformat it to date/time according to column definition
*/
public class DateConditionModifier implements TupleFilterSerializer.Decorator {
- private IdentityHashMap<TupleFilter, Boolean> dateCompareTupleChildren;
+ private IdentityHashMap<TupleFilter, DataType> dateCompareTupleChildren;
public DateConditionModifier(TupleFilter root) {
this.dateCompareTupleChildren = Maps.newIdentityHashMap();
}
- /**
- * replace filter on timestamp column to null, so that two tuple filter trees can
- * be compared regardless of the filter condition on timestamp column (In top level where conditions concatenated by ANDs)
- * @param filter
- * @return
- */
@Override
public TupleFilter onSerialize(TupleFilter filter) {
@@ -38,24 +35,39 @@ public class DateConditionModifier implements TupleFilterSerializer.Decorator {
}
TblColRef col = cfilter.getColumn();
- if (col == null || (!"date".equals(col.getDatatype()))) {
+ if (col == null || !col.getType().isDateTimeFamily()) {
return cfilter;
}
for (TupleFilter child : filter.getChildren()) {
- dateCompareTupleChildren.put(child, true);
+ dateCompareTupleChildren.put(child, col.getType());
}
}
if (filter instanceof ConstantTupleFilter && dateCompareTupleChildren.containsKey(filter)) {
ConstantTupleFilter constantTupleFilter = (ConstantTupleFilter) filter;
Set<String> newValues = Sets.newHashSet();
+ DataType columnType = dateCompareTupleChildren.get(filter);
for (String value : (Collection<String>) constantTupleFilter.getValues()) {
- newValues.add(DateFormat.formatToDateStr(Long.valueOf(value)));
+ newValues.add(formatTime(Long.valueOf(value), columnType));
}
return new ConstantTupleFilter(newValues);
}
return filter;
}
+
+ private String formatTime(long millis, DataType dataType) {
+ if (dataType.isDatetime() || dataType.isTime()) {
+ throw new RuntimeException("Datetime and time type are not supported yet");
+ }
+
+ if (dataType.isTimestamp()) {
+ return DateFormat.formatToTimeStr(millis);
+ } else if (dataType.isDate()) {
+ return DateFormat.formatToDateStr(millis);
+ } else {
+ throw new RuntimeException("Unknown type " + dataType + " to formatTime");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4219fdac/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java b/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
index 607b547..027b348 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
@@ -89,7 +89,6 @@ public class DataType {
private static final ConcurrentMap<DataType, DataType> CACHE = new ConcurrentHashMap<DataType, DataType>();
-
public static DataType getInstance(String type) {
if (type == null)
return null;
@@ -144,7 +143,6 @@ public class DataType {
}
}
-
// FIXME 256 for unknown string precision
if ((name.equals("char") || name.equals("varchar")) && precision == -1) {
precision = 256; // to save memory at frontend, e.g. tableau will
@@ -203,11 +201,22 @@ public class DataType {
return DATETIME_FAMILY.contains(name);
}
- public boolean isDate()
- {
+ public boolean isDate() {
return name.equals("date");
}
+ public boolean isTime() {
+ return name.equals("time");
+ }
+
+ public boolean isTimestamp() {
+ return name.equals("timestamp");
+ }
+
+ public boolean isDatetime() {
+ return name.equals("datetime");
+ }
+
public boolean isTinyInt() {
return name.equals("tinyint");
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4219fdac/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 04680d1..e580f3a 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -164,7 +164,7 @@ public class CacheService extends BasicService {
if (Constant.SERVER_MODE_JOB.equals(serverMode.toLowerCase()) || Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase())) {
logger.debug("This is the job engine node, will check whether auto merge is needed on cube " + cubeName);
CubeSegment newSeg = null;
- synchronized (getCubeManager().getCube(cubeName)) {
+ synchronized (CacheService.class) {
CubeInstance cube = getCubeManager().getCube(cubeName);
try {
newSeg = getCubeManager().autoMergeCubeSegments(cube);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4219fdac/streaming/src/main/java/org/apache/kylin/streaming/KafkaClusterConfig.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/KafkaClusterConfig.java b/streaming/src/main/java/org/apache/kylin/streaming/KafkaClusterConfig.java
index 0661498..ff34dd9 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/KafkaClusterConfig.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/KafkaClusterConfig.java
@@ -14,7 +14,6 @@ import javax.annotation.Nullable;
import java.util.List;
/**
- * Created by Hongbin Ma(Binmahone) on 6/3/15.
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class KafkaClusterConfig extends RootPersistentEntity {