You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/05/15 05:06:32 UTC
[20/50] [abbrv] incubator-kylin git commit: KYLIN-749 added
TimeStrDictionary for non date types.
KYLIN-749 added TimeStrDictionary for non date types.
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/9854f7d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/9854f7d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/9854f7d3
Branch: refs/heads/streaming-localdict
Commit: 9854f7d3cdc88e7af146e97956a165108547a041
Parents: 05e0bad
Author: honma <ho...@ebay.com>
Authored: Thu May 7 15:00:56 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu May 7 15:04:48 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/BasicTest.java | 6 +-
.../java/org/apache/kylin/dict/Dictionary.java | 9 +-
.../apache/kylin/dict/DictionaryGenerator.java | 4 +-
.../apache/kylin/dict/DictionaryManager.java | 4 +-
.../apache/kylin/dict/TimeStrDictionary.java | 115 +++++++++++++++++++
.../kylin/dict/TimeStrDictionaryTests.java | 61 ++++++++++
.../invertedindex/index/TableRecordInfo.java | 6 +-
.../invertedindex/model/IIKeyValueCodec.java | 2 +-
.../job/hadoop/cube/NewBaseCuboidMapper.java | 4 +-
.../job/hadoop/cube/MergeCuboidMapperTest.java | 2 +-
.../metadata/filter/ColumnTupleFilter.java | 14 +--
.../metadata/filter/CompareTupleFilter.java | 19 +--
.../metadata/filter/ConstantTupleFilter.java | 7 +-
.../apache/kylin/metadata/model/TblColRef.java | 2 +-
.../kylin/metadata/project/ProjectL2Cache.java | 4 +-
.../metadata/realization/SQLDigestUtil.java | 11 +-
.../kylin/query/relnode/OLAPFilterRel.java | 10 +-
.../apache/kylin/query/test/KylinQueryTest.java | 2 +-
.../storage/gridtable/GTFilterScanner.java | 2 +-
.../apache/kylin/storage/gridtable/GTInfo.java | 2 +-
.../storage/gridtable/GTInvertedIndex.java | 2 +-
.../storage/gridtable/GTScanRangePlanner.java | 2 +-
.../kylin/storage/gridtable/GTScanRequest.java | 4 +-
.../apache/kylin/storage/gridtable/GTUtil.java | 2 +-
.../kylin/storage/hbase/CubeTupleConverter.java | 2 +-
.../storage/hbase/DerivedFilterTranslator.java | 4 +-
.../hbase/coprocessor/CoprocessorFilter.java | 4 +-
.../DateConditionInplaceModifier.java | 63 ++++++++++
.../endpoint/EndpointTupleIterator.java | 21 ++--
.../org/apache/kylin/storage/tuple/Tuple.java | 14 +--
.../DateConditionInplaceModifierTest.java | 34 ++++++
.../apache/kylin/storage/test/StorageTest.java | 2 +-
32 files changed, 359 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index db80934..efa1459 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -63,8 +63,9 @@ public class BasicTest {
@Test
@Ignore("convenient trial tool for dev")
public void test1() throws Exception {
- System.out.println(Long.parseLong("0032"));
- System.out.println(time(1000L * Integer.MAX_VALUE));
+ System.out.println(time(946684800000L));
+
+ System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1000L * Integer.MAX_VALUE));
System.out.println(Ranges.open(3, 5).isConnected(Ranges.open(3, 10)));
@@ -80,7 +81,6 @@ public class BasicTest {
System.out.println("n");
System.out.println("The start key is set to " + null);
- System.out.println(time(946684800000L));
long current = System.currentTimeMillis();
System.out.println(time(current));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
index d6addb5..070e9de 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
@@ -18,13 +18,12 @@
package org.apache.kylin.dict;
-import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-
import org.apache.hadoop.io.Writable;
-
import org.apache.kylin.common.util.BytesUtil;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+
/**
* A bi-way dictionary that maps from dimension/column values to IDs and vice
* versa. By storing IDs instead of real values, the size of cube is
@@ -45,7 +44,7 @@ abstract public class Dictionary<T> implements Writable {
public static final byte NULL = (byte) 0xff;
// ID with all bit-1 (0xff e.g.) reserved for NULL value
- public static final int NULL_ID[] = new int[] { 0, 0xff, 0xffff, 0xffffff, 0xffffff };
+ public static final int NULL_ID[] = new int[] { 0, 0xff, 0xffff, 0xffffff, 0xffffffff };
abstract public int getMinId();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 59d7bb3..467456a 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -147,8 +147,8 @@ public class DictionaryGenerator {
}
}
- //FIXME: except for date type, all other date time family types are treated as string
- return buildStringDict(values, baseId, nSamples, samples);
+ //FIXME: except for date type, all other date time family types are treated as TimeStrDictionary
+ return new TimeStrDictionary();
}
private static Dictionary buildStringDict(Collection<byte[]> values, int baseId, int nSamples, ArrayList samples) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 8e5d599..fdedaf3 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -194,7 +194,7 @@ public class DictionaryManager {
if (model == null) {
srcTable = col.getTable();
srcCol = col.getName();
- srcColIdx = col.getColumn().getZeroBasedIndex();
+ srcColIdx = col.getColumnDesc().getZeroBasedIndex();
int nColumns = metaMgr.getTableDesc(col.getTable()).getColumnCount();
table = new FileTable(factColumnsPath + "/" + col.getName(), nColumns);
return new Object[] { srcTable, srcCol, srcColIdx, table };
@@ -220,7 +220,7 @@ public class DictionaryManager {
}
srcTable = col.getTable();
srcCol = col.getName();
- srcColIdx = col.getColumn().getZeroBasedIndex();
+ srcColIdx = col.getColumnDesc().getZeroBasedIndex();
if (model.isFactTable(col.getTable())) {
table = new FileTable(factColumnsPath + "/" + col.getName(), -1);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
new file mode 100644
index 0000000..aef26ec
--- /dev/null
+++ b/dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
@@ -0,0 +1,115 @@
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.DateFormat;
+
+import java.io.*;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/6/15.
+ */
+public class TimeStrDictionary extends Dictionary<String> {
+
+ private long maxid = Integer.MAX_VALUE;
+ private int maxLenghOfPositiveLong = 19;
+
+ @Override
+ public int getMinId() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxId() {
+ return Integer.MAX_VALUE;
+ }
+
+ @Override
+ public int getSizeOfId() {
+ return 4;
+ }
+
+ @Override
+ public int getSizeOfValue() {
+ return maxLenghOfPositiveLong;
+ }
+
+ @Override
+ protected int getIdFromValueImpl(String value, int roundingFlag) {
+ long millis = DateFormat.stringToMillis(value);
+ long seconds = millis / 1000;
+
+ if (seconds > maxid) {
+ return nullId();
+ } else if (seconds < 0) {
+ throw new IllegalArgumentException("Illegal value: " + value + ", parsed seconds: " + seconds);
+ }
+
+ return (int) seconds;
+ }
+
+ /**
+ *
+ * @param id
+ * @return return like "0000001430812800000"
+ */
+ @Override
+ protected String getValueFromIdImpl(int id) {
+ if (id == nullId())
+ return null;
+
+ long millis = 1000L * id;
+ String s = Long.toString(millis);
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < maxLenghOfPositiveLong - s.length(); ++i) {
+ sb.append('0');
+ }
+ sb.append(s);
+ return sb.toString();
+ }
+
+ @Override
+ final protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
+ try {
+ return getIdFromValue(new String(value, offset, len, "ISO-8859-1"));
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e); // never happen
+ }
+ }
+
+ @Override
+ final protected byte[] getValueBytesFromIdImpl(int id) {
+ String date = getValueFromId(id);
+ byte bytes[];
+ try {
+ bytes = date.getBytes("ISO-8859-1");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e); // never happen
+ }
+ return bytes;
+ }
+
+ @Override
+ final protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
+ byte bytes[] = getValueBytesFromIdImpl(id);
+ System.arraycopy(bytes, 0, returnValue, offset, bytes.length);
+ return bytes.length;
+ }
+
+ @Override
+ public void dump(PrintStream out) {
+ out.println(this.toString());
+ }
+
+ @Override
+ public String toString() {
+ return "TimeStrDictionary supporting from 1970-01-01 00:00:00 to 2038/01/19 03:14:07 (does not support millisecond)";
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java b/dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java
new file mode 100644
index 0000000..e723126
--- /dev/null
+++ b/dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTests.java
@@ -0,0 +1,61 @@
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.DateFormat;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Date;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/6/15.
+ */
+public class TimeStrDictionaryTests {
+ TimeStrDictionary dict;
+
+ @Before
+ public void setup() {
+ dict = new TimeStrDictionary();
+ }
+
+ @Test
+ public void basicTest() {
+ int a = dict.getIdFromValue("1999-01-01");
+ int b = dict.getIdFromValue("1999-01-01 00:00:00");
+ int c = dict.getIdFromValue("1999-01-01 00:00:00.000");
+ int d = dict.getIdFromValue("1999-01-01 00:00:00.022");
+
+ Assert.assertEquals(a, b);
+ Assert.assertEquals(a, c);
+ Assert.assertEquals(a, d);
+ }
+
+ @Test
+ public void testEncodeDecode() {
+ encodeDecode("1999-01-12", DateFormat.DEFAULT_DATE_PATTERN);
+ encodeDecode("2038-01-09", DateFormat.DEFAULT_DATE_PATTERN);
+ encodeDecode("2038-01-08", DateFormat.DEFAULT_DATE_PATTERN);
+ encodeDecode("1970-01-01", DateFormat.DEFAULT_DATE_PATTERN);
+ encodeDecode("1970-01-02", DateFormat.DEFAULT_DATE_PATTERN);
+
+ encodeDecode("1999-01-12 11:00:01", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ encodeDecode("2038-01-09 01:01:02", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ encodeDecode("2038-01-19 03:14:07", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ encodeDecode("1970-01-01 23:22:11", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ encodeDecode("1970-01-02 23:22:11", DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ }
+
+ @Test
+ public void testIllegal() {
+ Assert.assertEquals(-1, dict.getIdFromValue("2038-01-19 03:14:08"));
+ }
+
+ public void encodeDecode(String origin, String pattern) {
+ int a = dict.getIdFromValue(origin);
+ String v = dict.getValueFromId(a);
+ long ts = Long.parseLong(v);
+ String back = DateFormat.dateToString(new Date(ts), pattern);
+ Assert.assertEquals(origin, back);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index c57e91f..052b6bb 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -71,16 +71,16 @@ public class TableRecordInfo {
isMetric[i] = desc.isMetricsCol(i);
dataTypes[i] = tblColRef.getDatatype();
if (isMetric[i]) {
- lengths[i] = FixedLenMeasureCodec.get(DataType.getInstance(tblColRef.getColumn().getDatatype())).getLength();
+ lengths[i] = FixedLenMeasureCodec.get(DataType.getInstance(tblColRef.getColumnDesc().getDatatype())).getLength();
} else {
if (Array.isEmpty(dictionaryMap)) {
- final DataType dataType = DataType.getInstance(tblColRef.getColumn().getDatatype());
+ final DataType dataType = DataType.getInstance(tblColRef.getColumnDesc().getDatatype());
if (dataType.isNumberFamily()) {
lengths[i] = 16;
} else if (dataType.isStringFamily()) {
lengths[i] = 256;
} else if (dataType.isDateTimeFamily()) {
- lengths[i] = 23;
+ lengths[i] = 19;
} else {
throw new RuntimeException("invalid data type:" + dataType);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 4300140..8dbaed7 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -143,7 +143,7 @@ public class IIKeyValueCodec implements KeyValueCodec {
} else if (dataType.isStringFamily()) {
lengths[i] = 256;
} else if (dataType.isDateTimeFamily()) {
- lengths[i] = 23;
+ lengths[i] = 19;
} else {
throw new RuntimeException("invalid data type:" + dataType);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
index ae563ac..b4a93a1 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
@@ -178,7 +178,7 @@ public class NewBaseCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Text, Text, T
// create join infos
LinkedList<Integer> fkIndice = new LinkedList<Integer>();
for (TblColRef colRef : join.getForeignKeyColumns()) {
- fkIndice.add(colRef.getColumn().getZeroBasedIndex());
+ fkIndice.add(colRef.getColumnDesc().getZeroBasedIndex());
}
this.tableJoins.add(new TableJoin(joinType, fkIndice, lookupTableName, this.findColumnRowKeyRelationships(dim)));
@@ -200,7 +200,7 @@ public class NewBaseCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Text, Text, T
private LinkedList<Pair<Integer, Integer>> findColumnRowKeyRelationships(DimensionDesc dim) {
LinkedList<Pair<Integer, Integer>> dimTblColAsRowKey = new LinkedList<Pair<Integer, Integer>>();
for (TblColRef colRef : dim.getColumnRefs()) {
- int dimTableIndex = colRef.getColumn().getZeroBasedIndex();
+ int dimTableIndex = colRef.getColumnDesc().getZeroBasedIndex();
int rowKeyIndex = cubeDesc.getRowkey().getRowKeyIndexByColumnName(colRef.getName());
dimTblColAsRowKey.add(new Pair<Integer, Integer>(dimTableIndex, rowKeyIndex));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
index b5818c4..1d0b16b 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
@@ -121,7 +121,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
signature.setLastModifiedTime(System.currentTimeMillis());
signature.setPath("fake_dict_for" + lfn.getName() + segment.getName());
- DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumn().getName(), lfn.getColumn().getZeroBasedIndex(), "string", signature, "");
+ DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumnDesc().getName(), lfn.getColumnDesc().getZeroBasedIndex(), "string", signature, "");
List<byte[]> values = new ArrayList<byte[]>();
values.add(new byte[] { 97, 97, 97 });
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index fde41b1..309b6bb 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -18,18 +18,18 @@
package org.apache.kylin.metadata.filter;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
/**
*
* @author xjiang
@@ -89,7 +89,7 @@ public class ColumnTupleFilter extends TupleFilter {
String table = columnRef.getTable();
BytesUtil.writeUTFString(table, buffer);
- String columnId = columnRef.getColumn().getId();
+ String columnId = columnRef.getColumnDesc().getId();
BytesUtil.writeUTFString(columnId, buffer);
String columnName = columnRef.getName();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 57b50b7..fc7fd2c 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -18,17 +18,13 @@
package org.apache.kylin.metadata.filter;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import java.nio.ByteBuffer;
+import java.util.*;
+
/**
* @author xjiang
*/
@@ -94,6 +90,15 @@ public class CompareTupleFilter extends TupleFilter {
return conditionValues;
}
+ public void updateValues(Collection<?> values, boolean clear) {
+ if (clear) {
+ this.conditionValues.clear();
+ }
+
+ this.conditionValues.addAll(values);
+ this.firstCondValue = this.conditionValues.iterator().next();
+ }
+
public Object getFirstValue() {
return firstCondValue;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
index cc3add2..2cb6e8e 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
@@ -18,14 +18,14 @@
package org.apache.kylin.metadata.filter;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-
/**
*
* @author xjiang
@@ -78,6 +78,7 @@ public class ConstantTupleFilter extends TupleFilter {
return this.constantValues;
}
+
@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public byte[] serialize(IFilterCodeSystem cs) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 76c3911..cbc6ce5 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -67,7 +67,7 @@ public class TblColRef {
this.column = column;
}
- public ColumnDesc getColumn() {
+ public ColumnDesc getColumnDesc() {
return column;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 2fa7898..aa2ba30 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -208,7 +208,7 @@ class ProjectL2Cache {
return false;
}
ColumnDesc foundCol = table.findColumnByName(col.getName());
- if (col.getColumn().equals(foundCol) == false) {
+ if (col.getColumnDesc().equals(foundCol) == false) {
logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but it is not equal to '" + foundCol + "' according to MetadataManager");
return false;
}
@@ -239,7 +239,7 @@ class ProjectL2Cache {
TableCache tableCache = prjCache.tables.get(col.getTable());
prjCache.exposedTables.add(tableCache.tableDesc);
tableCache.exposed = true;
- tableCache.exposedColumns.add(col.getColumn());
+ tableCache.exposedColumns.add(col.getColumnDesc());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
index 969259d..695ac8f 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
@@ -1,12 +1,13 @@
package org.apache.kylin.metadata.realization;
-import com.google.common.base.Function;
-import com.google.common.collect.BoundType;
-import com.google.common.collect.Range;
+import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.metadata.filter.*;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.common.util.DateFormat;
+
+import com.google.common.base.Function;
+import com.google.common.collect.BoundType;
+import com.google.common.collect.Range;
/**
* Created by Hongbin Ma(Binmahone) on 4/14/15.
@@ -59,7 +60,7 @@ public class SQLDigestUtil {
}
private static TupleFilter createFilterForRealtime(TupleFilter originFilter, TblColRef partitionColRef, Range<Long> tsRange) {
- DataType type = partitionColRef.getColumn().getType();
+ DataType type = partitionColRef.getColumnDesc().getType();
String startTimeStr, endTimeStr;
CompareTupleFilter startFilter = null, endFilter = null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index f15078a..6f8f0f7 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -29,6 +29,7 @@ import net.hydromatic.optiq.runtime.SqlFunctions;
import org.apache.kylin.metadata.filter.*;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.hbase.coprocessor.DateConditionInplaceModifier;
import org.eigenbase.rel.FilterRelBase;
import org.eigenbase.rel.RelCollation;
import org.eigenbase.rel.RelNode;
@@ -39,7 +40,10 @@ import org.eigenbase.sql.SqlKind;
import org.eigenbase.sql.SqlOperator;
import org.eigenbase.util.NlsString;
-import java.util.*;
+import java.util.GregorianCalendar;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
/**
* @author xjiang
@@ -188,7 +192,8 @@ public class OLAPFilterRel extends FilterRelBase implements OLAPRel, EnumerableR
strValue = ((NlsString) literalValue).getValue();
} else if (literalValue instanceof GregorianCalendar) {
GregorianCalendar g = (GregorianCalendar) literalValue;
- strValue = "" + g.get(Calendar.YEAR) + "-" + normToTwoDigits(g.get(Calendar.MONTH) + 1) + "-" + normToTwoDigits(g.get(Calendar.DAY_OF_MONTH));
+ //strValue = "" + g.get(Calendar.YEAR) + "-" + normToTwoDigits(g.get(Calendar.MONTH) + 1) + "-" + normToTwoDigits(g.get(Calendar.DAY_OF_MONTH));
+ strValue = Long.toString(g.getTimeInMillis());
} else if (literalValue instanceof SqlFunctions.TimeUnitRange) {
// Extract(x from y) in where clause
strValue = ((SqlFunctions.TimeUnitRange) literalValue).name();
@@ -255,6 +260,7 @@ public class OLAPFilterRel extends FilterRelBase implements OLAPRel, EnumerableR
TupleFilterVisitor visitor = new TupleFilterVisitor(this.columnRowType, context);
context.filter = this.condition.accept(visitor);
+ DateConditionInplaceModifier.modify(context.filter);
context.filterColumns = collectColumns(context.filter);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
index 6a20bdb..e95da2c 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
@@ -95,7 +95,7 @@ public class KylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = "src/test/resources/query/sample.sql";
+ String queryFileName = "src/test/resources/query/sql_orderby/query01.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTFilterScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTFilterScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTFilterScanner.java
index ebb6ce9..6858a67 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTFilterScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTFilterScanner.java
@@ -24,7 +24,7 @@ public class GTFilterScanner implements IGTScanner {
this.oneTuple = new IEvaluatableTuple() {
@Override
public Object getValue(TblColRef col) {
- return next.get(col.getColumn().getZeroBasedIndex());
+ return next.get(col.getColumnDesc().getZeroBasedIndex());
}
};
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
index b33a596..954e464 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
@@ -104,7 +104,7 @@ public class GTInfo {
}
public void validateColRef(TblColRef ref) {
- TblColRef expected = colRef(ref.getColumn().getZeroBasedIndex());
+ TblColRef expected = colRef(ref.getColumnDesc().getZeroBasedIndex());
if (expected != ref)
throw new IllegalArgumentException();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
index 449b174..7c6abec 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
@@ -193,7 +193,7 @@ public class GTInvertedIndex {
}
private int col(CompareTupleFilter filter) {
- return filter.getColumn().getColumn().getZeroBasedIndex();
+ return filter.getColumn().getColumnDesc().getZeroBasedIndex();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
index fe67a19..d3418f7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
@@ -73,7 +73,7 @@ public class GTScanRangePlanner {
List<GTRecord> hbaseFuzzyKeys = Lists.newArrayList();
for (ColumnRange range : andDimRanges) {
- int col = range.column.getColumn().getZeroBasedIndex();
+ int col = range.column.getColumnDesc().getZeroBasedIndex();
if (info.primaryKey.get(col) == false)
continue;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
index 64e6b90..14af632 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
@@ -91,7 +91,7 @@ public class GTScanRequest {
// filter columns must belong to the table
info.validateColRef(col);
// filter columns must be returned to satisfy upper layer evaluation (calcite)
- columns.set(col.getColumn().getZeroBasedIndex());
+ columns.set(col.getColumnDesc().getZeroBasedIndex());
}
// un-evaluatable filter must be removed
@@ -102,7 +102,7 @@ public class GTScanRequest {
// columns in un-evaluatable filter must be returned without loss so upper layer can do final evaluation
if (hasAggregation()) {
for (TblColRef col : unevaluableColumns) {
- aggrGroupBy.set(col.getColumn().getZeroBasedIndex());
+ aggrGroupBy.set(col.getColumnDesc().getZeroBasedIndex());
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
index 2b77ea0..d4936c4 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
@@ -98,7 +98,7 @@ public class GTUtil {
newCompareFilter.addChild(new ColumnTupleFilter(externalCol));
Object firstValue = constValues.iterator().next();
- int col = colMapping == null ? externalCol.getColumn().getZeroBasedIndex() : colMapping.indexOf(externalCol);
+ int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex() : colMapping.indexOf(externalCol);
TupleFilter result;
ByteArray code;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/hbase/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeTupleConverter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeTupleConverter.java
index 0962ff3..4a6b767 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeTupleConverter.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeTupleConverter.java
@@ -163,7 +163,7 @@ public class CubeTupleConverter {
private int[] initDerivedColIdx() {
int[] idx = new int[deriveInfo.columns.length];
for (int i = 0; i < idx.length; i++) {
- idx[i] = deriveInfo.columns[i].getColumn().getZeroBasedIndex();
+ idx[i] = deriveInfo.columns[i].getColumnDesc().getZeroBasedIndex();
}
return idx;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
index 36a281f..743769f 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/DerivedFilterTranslator.java
@@ -65,11 +65,11 @@ public class DerivedFilterTranslator {
assert hostInfo.type == DeriveType.LOOKUP;
assert hostCols.length == pkCols.length;
- int di = derivedCol.getColumn().getZeroBasedIndex();
+ int di = derivedCol.getColumnDesc().getZeroBasedIndex();
int[] pi = new int[pkCols.length];
int hn = hostCols.length;
for (int i = 0; i < hn; i++) {
- pi[i] = pkCols[i].getColumn().getZeroBasedIndex();
+ pi[i] = pkCols[i].getColumnDesc().getZeroBasedIndex();
}
Set<Array<String>> satisfyingHostRecords = Sets.newHashSet();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
index feb36ca..b3b06a7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
@@ -18,8 +18,6 @@
package org.apache.kylin.storage.hbase.coprocessor;
-import java.util.Set;
-
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.dict.IDictionaryAware;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -27,6 +25,8 @@ import org.apache.kylin.metadata.filter.TupleFilterSerializer;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import java.util.Set;
+
/**
* @author yangli9
*/
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DateConditionInplaceModifier.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DateConditionInplaceModifier.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DateConditionInplaceModifier.java
new file mode 100644
index 0000000..8569058
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DateConditionInplaceModifier.java
@@ -0,0 +1,63 @@
+package org.apache.kylin.storage.hbase.coprocessor;
+
+import com.google.common.collect.Lists;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/7/15.
+ *
+ * For historical reasons, date types are encoded with DateStrDcitionary
+ * rather than TimeStrDictionary, so the constant in CompareTupleFilter should
+ * be corrected to "yyyy-MM-dd" format for date types
+ */
+public class DateConditionInplaceModifier {
+
+ public static void modify(TupleFilter filter) {
+
+ if (filter instanceof CompareTupleFilter) {
+ CompareTupleFilter cfilter = (CompareTupleFilter) filter;
+ List<? extends TupleFilter> children = cfilter.getChildren();
+
+ if (children == null || children.size() < 1) {
+ throw new IllegalArgumentException("Illegal compare filter: " + cfilter);
+ }
+
+ if (cfilter.getColumn() == null || (!"date".equals(cfilter.getColumn().getColumnDesc().getTypeName()))) {
+ return;
+ }
+
+ int nonConstantChild = 0;
+ boolean firstUpdate = true;
+ for (int i = 0; i < children.size(); ++i) {
+ if (children.get(i) instanceof ConstantTupleFilter) {
+ ConstantTupleFilter constantAsFilter = (ConstantTupleFilter) children.get(i);
+ Collection<?> values = constantAsFilter.getValues();
+ Collection<Object> newValues = Lists.newArrayList();
+ for (Object x : values) {
+ newValues.add(x == null ? null : DateFormat.formatToDateStr(Long.parseLong((String) x)));
+ }
+ cfilter.updateValues(newValues, firstUpdate);
+ firstUpdate=false;
+ } else {
+ nonConstantChild++;
+ }
+ }
+ if (nonConstantChild != 1) {
+ throw new IllegalArgumentException("Illegal compare filter: " + cfilter);
+ }
+
+ return;
+
+ } else {
+ for (TupleFilter child : filter.getChildren()) {
+ modify(child);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
index 106e6c5..6a7c14a 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
@@ -18,11 +18,9 @@
package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-import java.io.IOException;
-import java.util.*;
-
-import javax.annotation.Nullable;
-
+import com.google.common.base.Function;
+import com.google.common.collect.*;
+import com.google.protobuf.ByteString;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.lang3.SerializationUtils;
@@ -55,14 +53,9 @@ import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Range;
-import com.google.common.collect.Ranges;
-import com.google.common.collect.Sets;
-import com.google.protobuf.ByteString;
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.*;
/**
* Created by Hongbin Ma(Binmahone) on 12/2/14.
@@ -208,7 +201,7 @@ public class EndpointTupleIterator implements ITupleIterator {
functionDesc.setReturnType(iiDefaultHLLC);
functionDesc.setReturnDataType(DataType.getInstance(iiDefaultHLLC));
} else {
- functionDesc.setReturnType(column.getColumn().getType().toString());
+ functionDesc.setReturnType(column.getColumnDesc().getType().toString());
functionDesc.setReturnDataType(DataType.getInstance(functionDesc.getReturnType()));
}
functionDesc.getParameter().setColRefs(ImmutableList.of(column));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
index e89f8e0..7224cfe 100644
--- a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
+++ b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
@@ -18,16 +18,15 @@
package org.apache.kylin.storage.tuple;
-import java.math.BigDecimal;
-import java.util.Date;
-import java.util.List;
-
import net.sf.ehcache.pool.sizeof.annotations.IgnoreSizeOf;
-
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.List;
+
/**
* @author xjiang
*/
@@ -57,8 +56,7 @@ public class Tuple implements ITuple {
@Override
public ITuple makeCopy() {
Tuple ret = new Tuple(this.info);
- for(int i = 0 ; i < this.values.length; ++i)
- {
+ for (int i = 0; i < this.values.length; ++i) {
ret.values[i] = this.values[i];
}
return ret;
@@ -163,6 +161,8 @@ public class Tuple implements ITuple {
Date dateValue = DateFormat.stringToDate(strValue); // NOTE: forces GMT timezone
long millis = dateValue.getTime();
return millisToEpicDays(millis);// Optiq expects Integer instead of Long. by honma
+ } else if ("timestamp".equals(dataTypeName) || "datetime".equals(dataTypeName)) {
+ return Long.parseLong(strValue);
} else if ("tinyint".equals(dataTypeName)) {
return Byte.valueOf(strValue);
} else if ("short".equals(dataTypeName) || "smallint".equals(dataTypeName)) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionInplaceModifierTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionInplaceModifierTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionInplaceModifierTest.java
new file mode 100644
index 0000000..204e8c6
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/DateConditionInplaceModifierTest.java
@@ -0,0 +1,34 @@
+package org.apache.kylin.storage.filter;
+
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.hbase.coprocessor.DateConditionInplaceModifier;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/7/15.
+ */
+public class DateConditionInplaceModifierTest extends FilterBaseTest {
+ @Test
+ public void basicTest() {
+ TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
+ ColumnDesc c1 = ColumnDesc.mockup(t1, 2, "CAL_DT", "date");
+ TblColRef column = new TblColRef(c1);
+
+ CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ ColumnTupleFilter columnFilter = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter);
+ ConstantTupleFilter constantFilter = null;
+ constantFilter = new ConstantTupleFilter("946684800000");
+ compareFilter.addChild(constantFilter);
+
+ DateConditionInplaceModifier.modify(compareFilter);
+ Assert.assertEquals("2000-01-01",compareFilter.getFirstValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9854f7d3/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java b/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
index ee810ce..2a258df 100644
--- a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
@@ -163,7 +163,7 @@ public class StorageTest extends HBaseMetadataTestCase {
info.setField(col.getName(), col, idx++);
}
- TableDesc sourceTable = groups.get(0).getColumn().getTable();
+ TableDesc sourceTable = groups.get(0).getColumnDesc().getTable();
for (FunctionDesc func : aggregations) {
TblColRef col = new TblColRef(func.newFakeRewriteColumn(sourceTable));
info.setField(col.getName(), col, idx++);