You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/06/08 20:22:31 UTC
[08/11] incubator-kylin git commit: KYLIN-815
IPartitionConditionBuilder, flexible range condition to match hive partitions
KYLIN-815 IPartitionConditionBuilder, flexible range condition to match hive partitions
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1dc1a370
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1dc1a370
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1dc1a370
Branch: refs/heads/0.7
Commit: 1dc1a370dc0707925757d7dff4e2f05bf65b3be5
Parents: d2ae503
Author: Yang Li <li...@apache.org>
Authored: Sun Jun 7 16:35:34 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Jun 7 16:35:34 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/ClassUtil.java | 7 ++
.../apache/kylin/common/util/DateFormat.java | 101 +++++++++++++++++++
.../apache/kylin/dict/DateStrDictionary.java | 66 +-----------
.../kylin/invertedindex/index/TableRecord.java | 9 +-
.../org/apache/kylin/job/JoinedFlatTable.java | 50 +++------
.../hadoop/hive/CubeJoinedFlatTableDesc.java | 12 ++-
.../kylin/metadata/model/PartitionDesc.java | 79 +++++++++++----
.../kylin/storage/hbase/HBaseKeyRange.java | 14 +--
.../org/apache/kylin/storage/tuple/Tuple.java | 6 +-
9 files changed, 209 insertions(+), 135 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
index b474c58..4703e86 100644
--- a/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
@@ -50,4 +50,11 @@ public class ClassUtil {
return (Class<? extends T>) Class.forName(name);
}
+ public static Object newInstance(String clz) {
+ try {
+ return forName(clz, Object.class).newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
new file mode 100644
index 0000000..7aca923
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.util;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class DateFormat {
+
+ public static final String DEFAULT_DATE_PATTERN = "yyyy-MM-dd";
+ public static final String DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS = "yyyy-MM-dd HH:mm:ss";
+ public static final String DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS = "yyyy-MM-dd HH:mm:ss.SSS";
+
+ static final private Map<String, ThreadLocal<SimpleDateFormat>> threadLocalMap = new ConcurrentHashMap<String, ThreadLocal<SimpleDateFormat>>();
+
+ public static SimpleDateFormat getDateFormat(String datePattern) {
+ ThreadLocal<SimpleDateFormat> formatThreadLocal = threadLocalMap.get(datePattern);
+ if (formatThreadLocal == null) {
+ threadLocalMap.put(datePattern, formatThreadLocal = new ThreadLocal<SimpleDateFormat>());
+ }
+ SimpleDateFormat format = formatThreadLocal.get();
+ if (format == null) {
+ format = new SimpleDateFormat(datePattern);
+ format.setTimeZone(TimeZone.getTimeZone("GMT")); // NOTE: this must be GMT to calculate epoch date correctly
+ formatThreadLocal.set(format);
+ }
+ return format;
+ }
+
+ public static String formatToDateStr(long millis) {
+ return getDateFormat(DEFAULT_DATE_PATTERN).format(new Date(millis));
+ }
+
+ public static String formatToTimeStr(long millis) {
+ return getDateFormat(DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).format(new Date(millis));
+ }
+
+ public static String dateToString(Date date) {
+ return dateToString(date, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ }
+
+ public static String dateToString(Date date, String pattern) {
+ return getDateFormat(pattern).format(date);
+ }
+
+ public static Date stringToDate(String str) {
+ return stringToDate(str, DEFAULT_DATE_PATTERN);
+ }
+
+ public static Date stringToDate(String str, String pattern) {
+ Date date = null;
+ try {
+ date = getDateFormat(pattern).parse(str);
+ } catch (ParseException e) {
+ throw new IllegalArgumentException("'" + str + "' is not a valid date of pattern '" + pattern + "'", e);
+ }
+ return date;
+ }
+
+ public static long stringToMillis(String str) {
+ if (isAllDigits(str)) {
+ return Long.parseLong(str);
+ } else if (str.length() == 10) {
+ return stringToDate(str, DEFAULT_DATE_PATTERN).getTime();
+ } else if (str.length() == 19) {
+ return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS).getTime();
+ } else if (str.length() == 23) {
+ return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).getTime();
+ } else {
+ throw new IllegalArgumentException("there is no valid date pattern for:" + str);
+ }
+ }
+
+ private static boolean isAllDigits(String str) {
+ for (int i = 0, n = str.length(); i < n; i++) {
+ if (Character.isDigit(str.charAt(i)) == false)
+ return false;
+ }
+ return true;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
index 7cace15..ab4915a 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
@@ -23,14 +23,10 @@ import java.io.DataOutput;
import java.io.IOException;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.util.Date;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.DateFormat;
/**
* A dictionary for date string (date only, no time).
@@ -44,62 +40,8 @@ import org.apache.commons.lang.StringUtils;
*/
public class DateStrDictionary extends Dictionary<String> {
- static final String DEFAULT_DATE_PATTERN = "yyyy-MM-dd";
- static final String DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS = "yyyy-MM-dd HH:mm:ss";
- static final String DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS = "yyyy-MM-dd HH:mm:ss.SSS";
-
static final int ID_9999_12_31 = 3652426; // assume 0 based
- static final private Map<String, ThreadLocal<SimpleDateFormat>> threadLocalMap = new ConcurrentHashMap<String, ThreadLocal<SimpleDateFormat>>();
-
- static SimpleDateFormat getDateFormat(String datePattern) {
- ThreadLocal<SimpleDateFormat> formatThreadLocal = threadLocalMap.get(datePattern);
- if (formatThreadLocal == null) {
- threadLocalMap.put(datePattern, formatThreadLocal = new ThreadLocal<SimpleDateFormat>());
- }
- SimpleDateFormat format = formatThreadLocal.get();
- if (format == null) {
- format = new SimpleDateFormat(datePattern);
- format.setTimeZone(TimeZone.getTimeZone("GMT")); // NOTE: this must be GMT to calculate epoch date correctly
- formatThreadLocal.set(format);
- }
- return format;
- }
-
- public static String dateToString(Date date) {
- return dateToString(date, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
- }
-
- public static String dateToString(Date date, String pattern) {
- return getDateFormat(pattern).format(date);
- }
-
- public static Date stringToDate(String str) {
- return stringToDate(str, DEFAULT_DATE_PATTERN);
- }
-
- public static Date stringToDate(String str, String pattern) {
- Date date = null;
- try {
- date = getDateFormat(pattern).parse(str);
- } catch (ParseException e) {
- throw new IllegalArgumentException("'" + str + "' is not a valid date of pattern '" + pattern + "'", e);
- }
- return date;
- }
-
- public static long stringToMillis(String str) {
- if (str.length() == 10) {
- return stringToDate(str, DEFAULT_DATE_PATTERN).getTime();
- } else if (str.length() == 19) {
- return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS).getTime();
- } else if (str.length() == 23) {
- return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).getTime();
- } else {
- throw new IllegalArgumentException("there is no valid date pattern for:" + str);
- }
- }
-
// ============================================================================
private String pattern;
@@ -107,7 +49,7 @@ public class DateStrDictionary extends Dictionary<String> {
private int maxId;
public DateStrDictionary() {
- init(DEFAULT_DATE_PATTERN, 0);
+ init(DateFormat.DEFAULT_DATE_PATTERN, 0);
}
public DateStrDictionary(String datePattern, int baseId) {
@@ -147,7 +89,7 @@ public class DateStrDictionary extends Dictionary<String> {
@Override
final protected int getIdFromValueImpl(String value, int roundFlag) {
- Date date = stringToDate(value, pattern);
+ Date date = DateFormat.stringToDate(value, pattern);
int id = calcIdFromSeqNo(getNumOfDaysSince0000(date));
if (id < baseId || id > maxId)
throw new IllegalArgumentException("'" + value + "' encodes to '" + id + "' which is out of range [" + baseId + "," + maxId + "]");
@@ -160,7 +102,7 @@ public class DateStrDictionary extends Dictionary<String> {
if (id < baseId || id > maxId)
throw new IllegalArgumentException("ID '" + id + "' is out of range [" + baseId + "," + maxId + "]");
Date d = getDateFromNumOfDaysSince0000(calcSeqNoFromId(id));
- return dateToString(d, pattern);
+ return DateFormat.dateToString(d, pattern);
}
private int getNumOfDaysSince0000(Date d) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
index 3b8d969..d10554b 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
@@ -18,13 +18,14 @@
package org.apache.kylin.invertedindex.index;
-import com.google.common.collect.Lists;
-import org.apache.kylin.dict.DateStrDictionary;
+import java.util.List;
+
import org.apache.commons.lang.ObjectUtils;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.io.LongWritable;
+import org.apache.kylin.common.util.DateFormat;
-import java.util.List;
+import com.google.common.collect.Lists;
/**
* @author yangli9, honma
@@ -66,7 +67,7 @@ public class TableRecord implements Cloneable {
public long getTimestamp() {
String str = getValueString(info.getTimestampColumn());
- return DateStrDictionary.stringToMillis(str);
+ return DateFormat.stringToMillis(str);
}
public int length(int col) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 7e714d0..db6dd0e 100644
--- a/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -20,19 +20,15 @@ package org.apache.kylin.job;
import java.io.File;
import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.w3c.dom.Document;
-import org.w3c.dom.NodeList;
-import org.xml.sax.SAXException;
-
-import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.job.engine.JobEngineConfig;
@@ -43,7 +39,11 @@ import org.apache.kylin.job.hadoop.hive.SqlHiveDataTypeMapping;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.LookupDesc;
+import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
/**
* @author George Song (ysong1)
@@ -144,7 +144,7 @@ public class JoinedFlatTable {
return sql.toString();
}
- private static Map buildTableAliasMap(DataModelDesc dataModelDesc) {
+ private static Map<String, String> buildTableAliasMap(DataModelDesc dataModelDesc) {
Map<String, String> tableAliasMap = new HashMap<String, String>();
tableAliasMap.put(dataModelDesc.getFactTable().toUpperCase(), FACT_TABLE_ALIAS);
@@ -207,36 +207,23 @@ public class JoinedFlatTable {
whereBuilder.append("WHERE");
CubeDesc cubeDesc = desc.getCubeDesc();
+ DataModelDesc model = cubeDesc.getModel();
- if (cubeDesc.getModel().getFilterCondition() != null && cubeDesc.getModel().getFilterCondition().equals("") == false) {
- whereBuilder.append(" (").append(cubeDesc.getModel().getFilterCondition()).append(") ");
+ if (model.getFilterCondition() != null && model.getFilterCondition().equals("") == false) {
+ whereBuilder.append(" (").append(model.getFilterCondition()).append(") ");
hasCondition = true;
}
CubeSegment cubeSegment = desc.getCubeSegment();
if (null != cubeSegment) {
+ PartitionDesc partDesc = model.getPartitionDesc();
long dateStart = cubeSegment.getDateRangeStart();
long dateEnd = cubeSegment.getDateRangeEnd();
if (!(dateStart == 0 && dateEnd == Long.MAX_VALUE)) {
- String partitionColumnName = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumn();
- int indexOfDot = partitionColumnName.lastIndexOf(".");
-
- // convert to use table alias;
- if (indexOfDot > 0) {
- String partitionTableName = partitionColumnName.substring(0, indexOfDot);
- String columeOnly = partitionColumnName.substring(indexOfDot);
- String partitionTableAlias = tableAliasMap.get(partitionTableName);
- partitionColumnName = partitionTableAlias + columeOnly;
- }
-
whereBuilder.append(hasCondition ? " AND (" : " (");
- if (dateStart > 0) {
- whereBuilder.append(partitionColumnName + " >= '" + formatDateTimeInWhereClause(dateStart) + "' ");
- whereBuilder.append("AND ");
- }
- whereBuilder.append(partitionColumnName + " < '" + formatDateTimeInWhereClause(dateEnd) + "'");
+ whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, dateStart, dateEnd, tableAliasMap));
whereBuilder.append(")\n");
hasCondition = true;
}
@@ -247,15 +234,6 @@ public class JoinedFlatTable {
}
}
- private static String formatDateTimeInWhereClause(long datetime) {
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- Date date = new Date(datetime);
- String str = f.format(date);
- // note "2014-10-01" >= "2014-10-01 00:00:00" is FALSE
- return StringUtil.dropSuffix(str, " 00:00:00");
- }
-
private static String colName(String canonicalColName) {
return canonicalColName.replace(".", "_");
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java b/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
index a972692..ae66a18 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
@@ -18,18 +18,20 @@
package org.apache.kylin.job.hadoop.hive;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import org.apache.kylin.common.util.BytesSplitter;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.metadata.model.*;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
/**
* @author George Song (ysong1)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index cf0a8e3..73db41f 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -20,15 +20,16 @@ package org.apache.kylin.metadata.model;
import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.StringSplitter;
+
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.util.StringSplitter;
/**
- * @author xduo
- *
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class PartitionDesc {
@@ -44,31 +45,37 @@ public class PartitionDesc {
private long partitionDateStart = 0L;
@JsonProperty("partition_type")
private PartitionType partitionType = PartitionType.APPEND;
+ @JsonProperty("partition_condition_builder")
+ private String partitionConditionBuilderClz = DefaultPartitionConditionBuilder.class.getName();
private TblColRef partitionDateColumnRef;
+ private IPartitionConditionBuilder partitionConditionBuilder;
public void init(Map<String, TableDesc> tables) {
- if (StringUtils.isNotEmpty(partitionDateColumn)) {
- partitionDateColumn = partitionDateColumn.toUpperCase();
+ if (StringUtils.isEmpty(partitionDateColumn))
+ return;
- String[] columns = StringSplitter.split(partitionDateColumn, ".");
+ partitionDateColumn = partitionDateColumn.toUpperCase();
- if (null != columns && columns.length == 3) {
- String tableName = columns[0].toUpperCase() + "." + columns[1].toUpperCase();
+ String[] columns = StringSplitter.split(partitionDateColumn, ".");
- TableDesc table = tables.get(tableName);
- ColumnDesc col = table.findColumnByName(columns[2]);
- if (col != null) {
- partitionDateColumnRef = new TblColRef(col);
- } else {
- throw new IllegalStateException("The column '" + partitionDateColumn + "' provided in 'partition_date_column' doesn't exist.");
- }
+ if (null != columns && columns.length == 3) {
+ String tableName = columns[0].toUpperCase() + "." + columns[1].toUpperCase();
+
+ TableDesc table = tables.get(tableName);
+ ColumnDesc col = table.findColumnByName(columns[2]);
+ if (col != null) {
+ partitionDateColumnRef = new TblColRef(col);
} else {
- throw new IllegalStateException("The 'partition_date_column' format is invalid: " + partitionDateColumn + ", it should be {db}.{table}.{column}.");
+ throw new IllegalStateException("The column '" + partitionDateColumn + "' provided in 'partition_date_column' doesn't exist.");
}
+ } else {
+ throw new IllegalStateException("The 'partition_date_column' format is invalid: " + partitionDateColumn + ", it should be {db}.{table}.{column}.");
}
+
+ partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
}
-
+
public boolean isPartitioned() {
return partitionDateColumnRef != null;
}
@@ -93,6 +100,10 @@ public class PartitionDesc {
return partitionType;
}
+ public IPartitionConditionBuilder getPartitionConditionBuilder() {
+ return partitionConditionBuilder;
+ }
+
public void setCubePartitionType(PartitionType partitionType) {
this.partitionType = partitionType;
}
@@ -101,4 +112,36 @@ public class PartitionDesc {
return partitionDateColumnRef;
}
+ // ============================================================================
+
+ public static interface IPartitionConditionBuilder {
+ String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive, Map<String, String> tableAlias);
+ }
+
+ public static class DefaultPartitionConditionBuilder implements IPartitionConditionBuilder {
+
+ @Override
+ public String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive, Map<String, String> tableAlias) {
+ String partitionColumnName = partDesc.getPartitionDateColumn();
+
+ // convert to use table alias
+ int indexOfDot = partitionColumnName.lastIndexOf(".");
+ if (indexOfDot > 0) {
+ String partitionTableName = partitionColumnName.substring(0, indexOfDot);
+ if (tableAlias != null && tableAlias.containsKey(partitionTableName))
+ partitionColumnName = tableAlias.get(partitionTableName) + partitionColumnName.substring(indexOfDot);
+ }
+
+ StringBuilder builder = new StringBuilder();
+
+ if (startInclusive > 0) {
+ builder.append(partitionColumnName + " >= '" + DateFormat.formatToDateStr(startInclusive) + "' ");
+ builder.append("AND ");
+ }
+ builder.append(partitionColumnName + " < '" + DateFormat.formatToDateStr(endExclusive) + "'");
+
+ return builder.toString();
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
index 7926b1d..cb2dce7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
@@ -25,11 +25,9 @@ import java.util.Map;
import java.util.Set;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.Pair;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
@@ -37,9 +35,11 @@ import org.apache.kylin.cube.kv.FuzzyKeyEncoder;
import org.apache.kylin.cube.kv.FuzzyMaskEncoder;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.dict.DateStrDictionary;
import org.apache.kylin.metadata.model.TblColRef;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
/**
*
* @author xjiang
@@ -145,10 +145,10 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
private void initPartitionRange(ColumnValueRange dimRange) {
if (null != dimRange.getBeginValue()) {
- this.partitionColumnStartDate = DateStrDictionary.stringToDate(dimRange.getBeginValue()).getTime();
+ this.partitionColumnStartDate = DateFormat.stringToDate(dimRange.getBeginValue()).getTime();
}
if (null != dimRange.getEndValue()) {
- this.partitionColumnEndDate = DateStrDictionary.stringToDate(dimRange.getEndValue()).getTime();
+ this.partitionColumnEndDate = DateFormat.stringToDate(dimRange.getEndValue()).getTime();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
index 96bdaa6..5023580 100644
--- a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
+++ b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
@@ -23,10 +23,10 @@ import java.util.Date;
import java.util.List;
import org.apache.kylin.common.util.Array;
+import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
-import org.apache.kylin.dict.DateStrDictionary;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
@@ -133,7 +133,7 @@ public class Tuple implements ITuple {
// TODO use data type enum instead of string comparison
if ("date".equals(dataType)) {
// convert epoch time
- Date dateValue = DateStrDictionary.stringToDate(strValue); // NOTE: forces GMT timezone
+ Date dateValue = DateFormat.stringToDate(strValue); // NOTE: forces GMT timezone
long millis = dateValue.getTime();
long days = millis / (1000 * 3600 * 24);
return Integer.valueOf((int) days); // Optiq expects Integer instead of Long. by honma
@@ -150,7 +150,7 @@ public class Tuple implements ITuple {
} else if ("decimal".equals(dataType)) {
return new BigDecimal(strValue);
} else if ("timestamp".equals(dataType)) {
- return Long.valueOf(DateStrDictionary.stringToMillis(strValue));
+ return Long.valueOf(DateFormat.stringToMillis(strValue));
} else if ("float".equals(dataType)){
return Float.valueOf(strValue);
} else {