You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/28 01:05:26 UTC
[49/50] incubator-kylin git commit: KYLIN-625,
filter constants convert pass
KYLIN-625, filter constants convert pass
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/48a79714
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/48a79714
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/48a79714
Branch: refs/heads/streaming-localdict
Commit: 48a797149b604d0f58f6b450bde2c4fc3c75937e
Parents: d7fc231
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Mar 27 21:03:13 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Mar 27 21:03:13 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/model/ColumnDesc.java | 10 +++++++
.../apache/kylin/metadata/model/TableDesc.java | 7 +++++
.../apache/kylin/storage/gridtable/GTUtil.java | 12 ++-------
.../storage/gridtable/DictGridTableTest.java | 28 ++++++++++++++++++--
4 files changed, 45 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/48a79714/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 95b320c..194b650 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.model;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+
import org.apache.commons.lang.StringUtils;
/**
@@ -131,4 +132,13 @@ public class ColumnDesc {
return "ColumnDesc [name=" + name + ",table=" + table.getIdentity() + "]";
}
+ public static ColumnDesc mockup(TableDesc table, int oneBasedColumnIndex, String name, String datatype) {
+ ColumnDesc desc = new ColumnDesc();
+ String id = "" + oneBasedColumnIndex;
+ desc.setId(id);
+ desc.setName(name);
+ desc.setDatatype(datatype);
+ desc.init(table);
+ return desc;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/48a79714/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index 6db1202..6934ae9 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -162,4 +162,11 @@ public class TableDesc extends RootPersistentEntity {
public String toString() {
return "TableDesc [database=" + getDatabase() + " name=" + name + "]";
}
+
+ /** create a mockup table for unit test */
+ public static TableDesc mockup(String tableName) {
+ TableDesc mockup = new TableDesc();
+ mockup.setName(tableName);
+ return mockup;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/48a79714/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
index 7d042eb..94e5206 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
@@ -19,18 +19,10 @@ import com.google.common.collect.Sets;
public class GTUtil {
- static final TableDesc MOCKUP_TABLE = new TableDesc();
- static {
- MOCKUP_TABLE.setName("GT_MOCKUP_TABLE");
- }
+ static final TableDesc MOCKUP_TABLE = TableDesc.mockup("GT_MOCKUP_TABLE");
static TblColRef tblColRef(int col, String datatype) {
- ColumnDesc desc = new ColumnDesc();
- String id = "" + (col + 1);
- desc.setId(id);
- desc.setName(id);
- desc.setDatatype(datatype);
- desc.init(MOCKUP_TABLE);
+ ColumnDesc desc = ColumnDesc.mockup(MOCKUP_TABLE, col + 1, "" + col, datatype);
return new TblColRef(desc);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/48a79714/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java
index 46ec66c..a3de8b8 100644
--- a/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java
@@ -21,7 +21,9 @@ import org.apache.kylin.metadata.filter.ExtractTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.gridtable.GTInfo.Builder;
import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore;
@@ -37,6 +39,7 @@ public class DictGridTableTest {
verifyFirstRow(table);
verifyScanWithUnevaluatableFilter(table);
verifyScanWithEvaluatableFilter(table);
+ verifyConvertFilterConstants(table);
}
private void verifyFirstRow(GridTable table) throws IOException {
@@ -51,8 +54,9 @@ public class DictGridTableTest {
LogicalTupleFilter filter = and(fcomp, funevaluatable);
GTScanRequest req = new GTScanRequest(info, null, setOf(0), setOf(3), new String[] { "sum" }, filter);
+
// note the unEvaluatable column 1 in filter is added to group by
- assertEquals("GTScanRequest [range=null-null, columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.1 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], [null]], aggrGroupBy={0, 1}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
+ assertEquals("GTScanRequest [range=null-null, columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], [null]], aggrGroupBy={0, 1}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
doScanAndVerify(table, req, "[1421280000000, 20, null, 20, null]");
}
@@ -65,12 +69,32 @@ public class DictGridTableTest {
LogicalTupleFilter filter = and(fcomp1, fcomp2);
GTScanRequest req = new GTScanRequest(info, null, setOf(0), setOf(3), new String[] { "sum" }, filter);
+
// note the evaluatable column 1 in filter is added to returned columns but not in group by
- assertEquals("GTScanRequest [range=null-null, columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.1 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.2 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
+ assertEquals("GTScanRequest [range=null-null, columns={0, 1, 3}, filterPushDown=AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString());
doScanAndVerify(table, req, "[1421280000000, 30, null, 30, null]", "[1421366400000, 20, null, 40, null]");
}
+ private void verifyConvertFilterConstants(GridTable table) {
+ GTInfo info = table.getInfo();
+
+ TableDesc extTable = TableDesc.mockup("ext");
+ TblColRef extColA = new TblColRef(ColumnDesc.mockup(extTable, 1, "A", "timestamp"));
+ TblColRef extColB = new TblColRef(ColumnDesc.mockup(extTable, 2, "B", "integer"));
+
+ CompareTupleFilter fcomp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
+ CompareTupleFilter fcomp2 = compare(extColB, FilterOperatorEnum.EQ, "10");
+ LogicalTupleFilter filter = and(fcomp1, fcomp2);
+
+ Map<TblColRef, Integer> colMapping = Maps.newHashMap();
+ colMapping.put(extColA, 0);
+ colMapping.put(extColB, 1);
+
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 EQ [\\x00]]", newFilter.toString());
+ }
+
private void doScanAndVerify(GridTable table, GTScanRequest req, String... verifyRows) throws IOException {
System.out.println(req);
IGTScanner scanner = table.scan(req);