You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/10/22 08:07:56 UTC
[4/5] incubator-kylin git commit: planner tested
planner tested
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ab2dd7c6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ab2dd7c6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ab2dd7c6
Branch: refs/heads/KYLIN-942
Commit: ab2dd7c667a92da77374533a7b44175f55db8dc0
Parents: 6f05c14
Author: honma <ho...@ebay.com>
Authored: Thu Oct 22 11:53:17 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu Oct 22 11:53:17 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/BytesUtilTest.java | 4 -
.../kylin/gridtable/GTScanRangePlanner.java | 137 ++++++++++-------
.../kylin/gridtable/DictGridTableTest.java | 148 +++++++++++++++----
.../hbase/cube/v2/CubeSegmentScanner.java | 1 +
4 files changed, 205 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab2dd7c6/core-common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
index 04694ba..79bc9f1 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
@@ -26,9 +26,6 @@ import java.util.Arrays;
import org.junit.Test;
-/**
- * by honma
- */
public class BytesUtilTest {
@Test
public void test() {
@@ -62,7 +59,6 @@ public class BytesUtilTest {
testWriteReadUnsignedInt(73503300, 4);
}
- @Test
public void testWriteReadUnsignedInt(int testInt, int length) {
ByteArray ba = new ByteArray(new byte[length]);
BytesUtil.writeUnsigned(testInt, length, ba.asBuffer());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab2dd7c6/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index 12c5b05..d860090 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -39,10 +39,9 @@ public class GTScanRangePlanner {
final private Pair<ByteArray, ByteArray> segmentStartAndEnd;
final private TblColRef partitionColRef;
- final private ComparatorEx<ByteArray> byteUnknownIsSmaller;
- final private ComparatorEx<ByteArray> byteUnknownIsBigger;
- final private ComparatorEx<GTRecord> recordUnknownIsSmaller;
- final private ComparatorEx<GTRecord> recordUnknownIsBigger;
+ final private RecordComparator rangeStartComparator;
+ final private RecordComparator rangeEndComparator;
+ final private RecordComparator rangeStartEndComparator;
public GTScanRangePlanner(GTInfo info, Pair<ByteArray, ByteArray> segmentStartAndEnd, TblColRef partitionColRef) {
this.info = info;
@@ -51,10 +50,12 @@ public class GTScanRangePlanner {
IGTComparator comp = info.codeSystem.getComparator();
- this.byteUnknownIsSmaller = byteComparatorTreatsUnknownSmaller(comp);
- this.byteUnknownIsBigger = byteComparatorTreatsUnknownBigger(comp);
- this.recordUnknownIsSmaller = recordComparatorTreatsUnknownSmaller(comp);
- this.recordUnknownIsBigger = recordComparatorTreatsUnknownBigger(comp);
+ //start key GTRecord compare to start key GTRecord
+ this.rangeStartComparator = getRangeStartComparator(comp);
+ //stop key GTRecord compare to stop key GTRecord
+ this.rangeEndComparator = getRangeEndComparator(comp);
+ //start key GTRecord compare to stop key GTRecord
+ this.rangeStartEndComparator = getRangeStartEndComparator(comp);
}
// return empty list meaning filter is always false
@@ -92,22 +93,11 @@ public class GTScanRangePlanner {
for (ColumnRange range : andDimRanges) {
if (partitionColRef != null && range.column.equals(partitionColRef)) {
- boolean rangeEndGTESegStart = true;
- if (segmentStartAndEnd.getFirst().array() != null && range.end.array() != null) {
- rangeEndGTESegStart = info.getCodeSystem().getComparator().compare(range.end, segmentStartAndEnd.getFirst()) >= 0;
- }
-
- if (!rangeEndGTESegStart) {
- return null;
- }
-
- boolean rangeStartLTESegEnd = true;
- if (segmentStartAndEnd.getSecond().array() != null && range.begin.array() != null) {
- rangeStartLTESegEnd = info.getCodeSystem().getComparator().compare(range.begin, segmentStartAndEnd.getSecond()) <= 0;
- }
-
- if (!rangeStartLTESegEnd) {
+ if (rangeStartEndComparator.comparator.compare(segmentStartAndEnd.getFirst(), range.end) <= 0 //
+ && rangeStartEndComparator.comparator.compare(range.begin, segmentStartAndEnd.getSecond()) < 0) {
+ //segment range is [Closed,Open)
+ } else {
return null;
}
}
@@ -257,7 +247,7 @@ public class GTScanRangePlanner {
Collections.sort(ranges, new Comparator<GTScanRange>() {
@Override
public int compare(GTScanRange a, GTScanRange b) {
- return recordUnknownIsSmaller.compare(a.pkStart, b.pkStart);
+ return rangeStartComparator.compare(a.pkStart, b.pkStart);
}
});
@@ -265,13 +255,12 @@ public class GTScanRangePlanner {
List<GTScanRange> mergedRanges = new ArrayList<GTScanRange>();
int mergeBeginIndex = 0;
GTRecord mergeEnd = ranges.get(0).pkEnd;
- for (int index = 0; index < ranges.size(); index++) {
+ for (int index = 1; index < ranges.size(); index++) {
GTScanRange range = ranges.get(index);
// if overlap, swallow it
- if (recordUnknownIsSmaller.min(range.pkStart, mergeEnd) == range.pkStart //
- || recordUnknownIsBigger.max(mergeEnd, range.pkStart) == mergeEnd) {
- mergeEnd = recordUnknownIsBigger.max(mergeEnd, range.pkEnd);
+ if (rangeStartEndComparator.compare(range.pkStart, mergeEnd) <= 0) {
+ mergeEnd = rangeEndComparator.max(mergeEnd, range.pkEnd);
continue;
}
@@ -281,7 +270,7 @@ public class GTScanRangePlanner {
// start new split
mergeBeginIndex = index;
- mergeEnd = recordUnknownIsBigger.max(mergeEnd, range.pkEnd);
+ mergeEnd = range.pkEnd;
}
// don't miss the last range
@@ -304,7 +293,7 @@ public class GTScanRangePlanner {
for (GTScanRange range : ranges) {
hasNonFuzzyRange = hasNonFuzzyRange || range.fuzzyKeys.isEmpty();
newFuzzyKeys.addAll(range.fuzzyKeys);
- end = recordUnknownIsBigger.max(end, range.pkEnd);
+ end = rangeEndComparator.max(end, range.pkEnd);
}
// if any range is non-fuzzy, then all fuzzy keys must be cleared
@@ -345,11 +334,11 @@ public class GTScanRangePlanner {
break;
case LT:
case LTE:
- end = byteUnknownIsBigger.max(values);
+ end = rangeEndComparator.comparator.max(values);
break;
case GT:
case GTE:
- begin = byteUnknownIsSmaller.min(values);
+ begin = rangeStartComparator.comparator.min(values);
break;
case NEQ:
case NOTIN:
@@ -374,8 +363,8 @@ public class GTScanRangePlanner {
begin = ByteArray.EMPTY;
end = ByteArray.EMPTY;
} else {
- begin = byteUnknownIsSmaller.min(valueSet);
- end = byteUnknownIsBigger.max(valueSet);
+ begin = rangeStartComparator.comparator.min(valueSet);
+ end = rangeEndComparator.comparator.max(valueSet);
}
}
@@ -423,14 +412,14 @@ public class GTScanRangePlanner {
return;
}
- this.begin = byteUnknownIsSmaller.max(this.begin, another.begin);
- this.end = byteUnknownIsBigger.min(this.end, another.end);
+ this.begin = rangeStartComparator.comparator.max(this.begin, another.begin);
+ this.end = rangeEndComparator.comparator.min(this.end, another.end);
}
private Set<ByteArray> filter(Set<ByteArray> equalValues, ByteArray beginValue, ByteArray endValue) {
Set<ByteArray> result = Sets.newHashSetWithExpectedSize(equalValues.size());
for (ByteArray v : equalValues) {
- if (byteUnknownIsSmaller.compare(beginValue, v) <= 0 && byteUnknownIsBigger.compare(v, endValue) <= 0) {
+ if (rangeStartEndComparator.comparator.compare(beginValue, v) <= 0 && rangeStartEndComparator.comparator.compare(v, endValue) <= 0) {
result.add(v);
}
}
@@ -487,42 +476,55 @@ public class GTScanRangePlanner {
}
}
- public static ComparatorEx<ByteArray> byteComparatorTreatsUnknownSmaller(final IGTComparator comp) {
- return new ComparatorEx<ByteArray>() {
+ public static RecordComparator getRangeStartComparator(final IGTComparator comp) {
+ return new RecordComparator(new ComparatorEx<ByteArray>() {
@Override
public int compare(ByteArray a, ByteArray b) {
if (a.array() == null) {
- return -1;
+ if (b.array() == null) {
+ return 0;
+ } else {
+ return -1;
+ }
} else if (b.array() == null) {
return 1;
} else {
return comp.compare(a, b);
}
}
- };
+ });
}
- public static ComparatorEx<ByteArray> byteComparatorTreatsUnknownBigger(final IGTComparator comp) {
- return new ComparatorEx<ByteArray>() {
+ public static RecordComparator getRangeEndComparator(final IGTComparator comp) {
+ return new RecordComparator(new ComparatorEx<ByteArray>() {
@Override
public int compare(ByteArray a, ByteArray b) {
if (a.array() == null) {
- return 1;
+ if (b.array() == null) {
+ return 0;
+ } else {
+ return 1;
+ }
} else if (b.array() == null) {
return -1;
} else {
return comp.compare(a, b);
}
}
- };
- }
-
- public static ComparatorEx<GTRecord> recordComparatorTreatsUnknownSmaller(IGTComparator comp) {
- return new RecordComparator(byteComparatorTreatsUnknownSmaller(comp));
+ });
}
- public static ComparatorEx<GTRecord> recordComparatorTreatsUnknownBigger(IGTComparator comp) {
- return new RecordComparator(byteComparatorTreatsUnknownBigger(comp));
+ public static RecordComparator getRangeStartEndComparator(final IGTComparator comp) {
+ return new AsymmetricRecordComparator(new ComparatorEx<ByteArray>() {
+ @Override
+ public int compare(ByteArray a, ByteArray b) {
+ if (a.array() == null || b.array() == null) {
+ return -1;
+ } else {
+ return comp.compare(a, b);
+ }
+ }
+ });
}
private static class RecordComparator extends ComparatorEx<GTRecord> {
@@ -538,7 +540,7 @@ public class GTScanRangePlanner {
assert a.maskForEqualHashComp() == b.maskForEqualHashComp();
ImmutableBitSet mask = a.maskForEqualHashComp();
- int comp = 0;
+ int comp;
for (int i = 0; i < mask.trueBitCount(); i++) {
int c = mask.trueBitAt(i);
comp = comparator.compare(a.cols[c], b.cols[c]);
@@ -548,4 +550,35 @@ public class GTScanRangePlanner {
return 0; // equals
}
}
+
+ /**
+ * asymmetric means compare(a,b) > 0 does not cause compare(b,a) < 0
+ * so min max functions will not bu supported
+ */
+ private static class AsymmetricRecordComparator extends RecordComparator {
+
+ AsymmetricRecordComparator(ComparatorEx<ByteArray> byteComparator) {
+ super(byteComparator);
+ }
+
+ public GTRecord min(Collection<GTRecord> v) {
+ throw new UnsupportedOperationException();
+ }
+
+ public GTRecord max(Collection<GTRecord> v) {
+ throw new UnsupportedOperationException();
+ }
+
+ public GTRecord min(GTRecord a, GTRecord b) {
+ throw new UnsupportedOperationException();
+ }
+
+ public GTRecord max(GTRecord a, GTRecord b) {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean between(GTRecord v, GTRecord start, GTRecord end) {
+ throw new UnsupportedOperationException();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab2dd7c6/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index 283c6e9..91e7e18 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -29,6 +29,7 @@ import java.util.Map;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.gridtable.CubeCodeSystem;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.NumberDictionaryBuilder;
@@ -48,6 +49,7 @@ import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.junit.Before;
import org.junit.Test;
import com.google.common.collect.Lists;
@@ -55,31 +57,112 @@ import com.google.common.collect.Maps;
public class DictGridTableTest {
+ private GridTable table;
+ private GTInfo info;
+ private CompareTupleFilter timeComp0;
+ private CompareTupleFilter timeComp1;
+ private CompareTupleFilter timeComp2;
+ private CompareTupleFilter timeComp3;
+ private CompareTupleFilter timeComp4;
+ private CompareTupleFilter timeComp5;
+ private CompareTupleFilter timeComp6;
+ private CompareTupleFilter ageComp1;
+ private CompareTupleFilter ageComp2;
+ private CompareTupleFilter ageComp3;
+ private CompareTupleFilter ageComp4;
+
+ @Before
+ public void setup() throws IOException {
+ table = newTestTable();
+ info = table.getInfo();
+
+ timeComp0 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-14"));
+ timeComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
+ timeComp2 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-13"));
+ timeComp3 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-15"));
+ timeComp4 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-15"));
+ timeComp5 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-15"));
+ timeComp6 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-14"));
+ ageComp1 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "10"));
+ ageComp2 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "20"));
+ ageComp3 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "30"));
+ ageComp4 = compare(info.colRef(1), FilterOperatorEnum.NEQ, enc(info, 1, "30"));
+ }
+
+ @Test
+ public void verifySegmentSkipping() {
+
+ ByteArray segmentStart = enc(info, 0, "2015-01-14");
+ ByteArray segmentStartX = enc(info, 0, "2015-01-14 00:00:00");//when partition col is dict encoded, time format will be free
+ ByteArray segmentEnd = enc(info, 0, "2015-01-15");
+ assertEquals(segmentStart, segmentStartX);
+
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0));
+
+ {
+ LogicalTupleFilter filter = and(timeComp0, ageComp1);
+ List<GTScanRange> r = planner.planScanRanges(filter);
+ assertEquals(1, r.size());//scan range are [close,close]
+ assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString());
+ assertEquals(1, r.get(0).fuzzyKeys.size());
+ assertEquals("[[10]]", r.get(0).fuzzyKeys.toString());
+ }
+ {
+ LogicalTupleFilter filter = and(timeComp2, ageComp1);
+ List<GTScanRange> r = planner.planScanRanges(filter);
+ assertEquals(0, r.size());
+ }
+ {
+ LogicalTupleFilter filter = and(timeComp4, ageComp1);
+ List<GTScanRange> r = planner.planScanRanges(filter);
+ assertEquals(0, r.size());
+ }
+ {
+ LogicalTupleFilter filter = and(timeComp5, ageComp1);
+ List<GTScanRange> r = planner.planScanRanges(filter);
+ assertEquals(0, r.size());
+ }
+ {
+ LogicalTupleFilter filter = or(and(timeComp2, ageComp1), and(timeComp1, ageComp1), and(timeComp6, ageComp1));
+ List<GTScanRange> r = planner.planScanRanges(filter);
+ assertEquals(1, r.size());
+ assertEquals("[1421193600000, 10]-[null, 10]", r.get(0).toString());
+ assertEquals("[[10], [1421193600000, 10]]", r.get(0).fuzzyKeys.toString());
+ }
+ {
+ LogicalTupleFilter filter = or(timeComp2, timeComp1, timeComp6);
+ List<GTScanRange> r = planner.planScanRanges(filter);
+ assertEquals(1, r.size());
+ assertEquals("[1421193600000, null]-[null, null]", r.get(0).toString());
+ assertEquals(0, r.get(0).fuzzyKeys.size());
+ }
+ }
+
@Test
- public void test() throws IOException {
- GridTable table = newTestTable();
- verifyScanRangePlanner(table);
- verifyFirstRow(table);
- verifyScanWithUnevaluatableFilter(table);
- verifyScanWithEvaluatableFilter(table);
- verifyConvertFilterConstants1(table);
- verifyConvertFilterConstants2(table);
- verifyConvertFilterConstants3(table);
- verifyConvertFilterConstants4(table);
+ public void verifySegmentSkipping2() {
+ ByteArray segmentEnd = enc(info, 0, "2015-01-15");
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0));
+
+ {
+ LogicalTupleFilter filter = and(timeComp0, ageComp1);
+ List<GTScanRange> r = planner.planScanRanges(filter);
+ assertEquals(1, r.size());//scan range are [close,close]
+ assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString());
+ assertEquals(1, r.get(0).fuzzyKeys.size());
+ assertEquals("[[10]]", r.get(0).fuzzyKeys.toString());
+ }
+
+ {
+ LogicalTupleFilter filter = and(timeComp5, ageComp1);
+ List<GTScanRange> r = planner.planScanRanges(filter);
+ assertEquals(0, r.size());//scan range are [close,close]
+ }
}
- private void verifyScanRangePlanner(GridTable table) {
- GTInfo info = table.getInfo();
- GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null);
+ @Test
+ public void verifyScanRangePlanner() {
- CompareTupleFilter timeComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
- CompareTupleFilter timeComp2 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-13"));
- CompareTupleFilter timeComp3 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-15"));
- CompareTupleFilter timeComp4 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-15"));
- CompareTupleFilter ageComp1 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "10"));
- CompareTupleFilter ageComp2 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "20"));
- CompareTupleFilter ageComp3 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "30"));
- CompareTupleFilter ageComp4 = compare(info.colRef(1), FilterOperatorEnum.NEQ, enc(info, 1, "30"));
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null);
// flatten or-and & hbase fuzzy value
{
@@ -124,11 +207,13 @@ public class DictGridTableTest {
}
}
- private void verifyFirstRow(GridTable table) throws IOException {
+ @Test
+ public void verifyFirstRow() throws IOException {
doScanAndVerify(table, new GTScanRequest(table.getInfo()), "[1421193600000, 30, Yang, 10, 10.5]");
}
- private void verifyScanWithUnevaluatableFilter(GridTable table) throws IOException {
+ @Test
+ public void verifyScanWithUnevaluatableFilter() throws IOException {
GTInfo info = table.getInfo();
CompareTupleFilter fComp = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
@@ -144,7 +229,8 @@ public class DictGridTableTest {
doScanAndVerify(table, req, "[1421280000000, 20, null, 20, null]");
}
- private void verifyScanWithEvaluatableFilter(GridTable table) throws IOException {
+ @Test
+ public void verifyScanWithEvaluatableFilter() throws IOException {
GTInfo info = table.getInfo();
CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14"));
@@ -159,7 +245,8 @@ public class DictGridTableTest {
doScanAndVerify(table, req, "[1421280000000, 20, null, 30, null]", "[1421366400000, 20, null, 40, null]");
}
- private void verifyConvertFilterConstants1(GridTable table) {
+ @Test
+ public void verifyConvertFilterConstants1() {
GTInfo info = table.getInfo();
TableDesc extTable = TableDesc.mockup("ext");
@@ -178,7 +265,8 @@ public class DictGridTableTest {
assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 EQ [\\x00]]", newFilter.toString());
}
- private void verifyConvertFilterConstants2(GridTable table) {
+ @Test
+ public void verifyConvertFilterConstants2() {
GTInfo info = table.getInfo();
TableDesc extTable = TableDesc.mockup("ext");
@@ -198,7 +286,8 @@ public class DictGridTableTest {
assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], NULL.GT_MOCKUP_TABLE.1 LT [\\x00]]", newFilter.toString());
}
- private void verifyConvertFilterConstants3(GridTable table) {
+ @Test
+ public void verifyConvertFilterConstants3() {
GTInfo info = table.getInfo();
TableDesc extTable = TableDesc.mockup("ext");
@@ -218,7 +307,8 @@ public class DictGridTableTest {
assertEquals("AND [NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], []]", newFilter.toString());
}
- private void verifyConvertFilterConstants4(GridTable table) {
+ @Test
+ public void verifyConvertFilterConstants4() {
GTInfo info = table.getInfo();
TableDesc extTable = TableDesc.mockup("ext");
@@ -252,7 +342,7 @@ public class DictGridTableTest {
scanner.close();
}
- private Object enc(GTInfo info, int col, String value) {
+ private ByteArray enc(GTInfo info, int col, String value) {
ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength());
info.codeSystem.encodeColumnValue(col, value, buf);
return ByteArray.copyOf(buf.array(), buf.arrayOffset(), buf.position());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab2dd7c6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index d49de56..4721cb6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -113,6 +113,7 @@ public class CubeSegmentScanner implements IGTScanner {
if ("date".equalsIgnoreCase(partitionColType)) {
value = DateFormat.formatToDateStr(ts);
} else if ("timestamp".equalsIgnoreCase(partitionColType)) {
+ //TODO: if partition col is not dict encoded, value's format may differ from expected. Though by default it is not the case
value = DateFormat.formatToTimeWithoutMilliStr(ts);
} else {
throw new RuntimeException("Type " + partitionColType + " is not valid partition column type");