You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/06/22 02:52:58 UTC
incubator-kylin git commit: minor,
remove IFilterCodeSystem from IGTCodeSystem
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8 bb8d6d603 -> 10f075d63
minor, remove IFilterCodeSystem from IGTCodeSystem
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/10f075d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/10f075d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/10f075d6
Branch: refs/heads/0.8
Commit: 10f075d6386451dc9c6a4385129f03a3bcd71d6f
Parents: bb8d6d6
Author: Yang Li <li...@apache.org>
Authored: Mon Jun 22 08:52:50 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Mon Jun 22 08:52:50 2015 +0800
----------------------------------------------------------------------
.../kylin/storage/cube/CubeCodeSystem.java | 23 +++----------
.../storage/gridtable/GTFilterScanner.java | 2 +-
.../storage/gridtable/GTInvertedIndex.java | 2 +-
.../gridtable/GTInvertedIndexOfColumn.java | 11 +++---
.../kylin/storage/gridtable/GTRecord.java | 11 +++---
.../storage/gridtable/GTSampleCodeSystem.java | 23 +++----------
.../storage/gridtable/GTScanRangePlanner.java | 29 ++++++++--------
.../apache/kylin/storage/gridtable/GTUtil.java | 35 ++++++++++++++++++--
.../kylin/storage/gridtable/IGTCodeSystem.java | 4 +--
.../kylin/storage/gridtable/IGTComparator.java | 15 +++++++++
10 files changed, 84 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/10f075d6/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
index dbfef87..3d3ed51 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeCodeSystem.java
@@ -12,11 +12,11 @@ import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.metadata.filter.IFilterCodeSystem;
import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.apache.kylin.metadata.serializer.DataTypeSerializer;
import org.apache.kylin.storage.gridtable.GTInfo;
import org.apache.kylin.storage.gridtable.IGTCodeSystem;
+import org.apache.kylin.storage.gridtable.IGTComparator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,7 +35,7 @@ public class CubeCodeSystem implements IGTCodeSystem {
private Map<Integer, Dictionary> dictionaryMap; // column index ==> dictionary of column
private Map<Integer, Integer> fixLenMap; // column index ==> fixed length of column
private Map<Integer, Integer> dependentMetricsMap;
- private IFilterCodeSystem<ByteArray> filterCS;
+ private IGTComparator comparator;
private DataTypeSerializer[] serializers;
public CubeCodeSystem(Map<Integer, Dictionary> dictionaryMap) {
@@ -68,7 +68,7 @@ public class CubeCodeSystem implements IGTCodeSystem {
}
}
- this.filterCS = new IFilterCodeSystem<ByteArray>() {
+ this.comparator = new IGTComparator() {
@Override
public boolean isNull(ByteArray code) {
// all 0xff is null
@@ -84,25 +84,12 @@ public class CubeCodeSystem implements IGTCodeSystem {
public int compare(ByteArray code1, ByteArray code2) {
return code1.compareTo(code2);
}
-
- @Override
- public void serialize(ByteArray code, ByteBuffer buffer) {
- if (code == null)
- BytesUtil.writeByteArray(null, 0, 0, buffer);
- else
- BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer);
- }
-
- @Override
- public ByteArray deserialize(ByteBuffer buffer) {
- return new ByteArray(BytesUtil.readByteArray(buffer));
- }
};
}
@Override
- public IFilterCodeSystem<ByteArray> getFilterCodeSystem() {
- return filterCS;
+ public IGTComparator getComparator() {
+ return comparator;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/10f075d6/storage/src/main/java/org/apache/kylin/storage/gridtable/GTFilterScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTFilterScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTFilterScanner.java
index 6858a67..d60e4ba 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTFilterScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTFilterScanner.java
@@ -63,7 +63,7 @@ public class GTFilterScanner implements IGTScanner {
if (next != null)
return true;
- IFilterCodeSystem<ByteArray> filterCodeSystem = getInfo().codeSystem.getFilterCodeSystem();
+ IFilterCodeSystem<ByteArray> filterCodeSystem = GTUtil.wrap(getInfo().codeSystem.getComparator());
while (inputIterator.hasNext()) {
next = inputIterator.next();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/10f075d6/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
index 4196528..ceb1463 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
@@ -37,7 +37,7 @@ public class GTInvertedIndex {
index = new GTInvertedIndexOfColumn[info.getColumnCount()];
for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
int c = colPreferIndex.trueBitAt(i);
- index[c] = new GTInvertedIndexOfColumn(info.codeSystem.getFilterCodeSystem());
+ index[c] = new GTInvertedIndexOfColumn(info.codeSystem.getComparator());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/10f075d6/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndexOfColumn.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndexOfColumn.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndexOfColumn.java
index 810a7a3..10a944c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndexOfColumn.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndexOfColumn.java
@@ -6,23 +6,22 @@ import java.util.NavigableMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.metadata.filter.IFilterCodeSystem;
import com.google.common.collect.Maps;
public class GTInvertedIndexOfColumn {
- final private IFilterCodeSystem<ByteArray> codeSystem;
+ final private IGTComparator comparator;
final private ReentrantReadWriteLock rwLock;
private int nBlocks;
private NavigableMap<ByteArray, ConciseSet> rangeIndex;
private ConciseSet nullIndex;
- public GTInvertedIndexOfColumn(IFilterCodeSystem<ByteArray> codeSystem) {
- this.codeSystem = codeSystem;
+ public GTInvertedIndexOfColumn(IGTComparator comparator) {
+ this.comparator = comparator;
this.rwLock = new ReentrantReadWriteLock();
- this.rangeIndex = Maps.newTreeMap(codeSystem);
+ this.rangeIndex = Maps.newTreeMap(comparator);
this.nullIndex = new ConciseSet();
}
@@ -30,7 +29,7 @@ public class GTInvertedIndexOfColumn {
rwLock.writeLock().lock();
try {
for (ByteArray code : codes) {
- if (codeSystem.isNull(code)) {
+ if (comparator.isNull(code)) {
nullIndex.add(blockId);
continue;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/10f075d6/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
index 23d65bc..fbb6171 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
@@ -1,13 +1,12 @@
package org.apache.kylin.storage.gridtable;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.filter.IFilterCodeSystem;
-
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.BitSet;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
+
public class GTRecord implements Comparable<GTRecord> {
final GTInfo info;
@@ -167,12 +166,12 @@ public class GTRecord implements Comparable<GTRecord> {
public int compareTo(GTRecord o) {
assert this.info == o.info;
assert this.maskForEqualHashComp == o.maskForEqualHashComp; // reference equal for performance
- IFilterCodeSystem<ByteArray> cs = info.codeSystem.getFilterCodeSystem();
+ IGTComparator comparator = info.codeSystem.getComparator();
int comp = 0;
for (int i = 0; i < maskForEqualHashComp.trueBitCount(); i++) {
int c = maskForEqualHashComp.trueBitAt(i);
- comp = cs.compare(cols[c], o.cols[c]);
+ comp = comparator.compare(cols[c], o.cols[c]);
if (comp != 0)
return comp;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/10f075d6/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
index 7d6a1cc..71a9962 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTSampleCodeSystem.java
@@ -3,9 +3,7 @@ package org.apache.kylin.storage.gridtable;
import java.nio.ByteBuffer;
import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.filter.IFilterCodeSystem;
import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.apache.kylin.metadata.serializer.DataTypeSerializer;
@@ -22,7 +20,7 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
private GTInfo info;
private DataTypeSerializer[] serializers;
- private IFilterCodeSystem<ByteArray> filterCS;
+ private IGTComparator comparator;
public GTSampleCodeSystem() {
}
@@ -36,7 +34,7 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
this.serializers[i] = DataTypeSerializer.create(info.colTypes[i]);
}
- this.filterCS = new IFilterCodeSystem<ByteArray>() {
+ this.comparator = new IGTComparator() {
@Override
public boolean isNull(ByteArray code) {
// all 0xff is null
@@ -52,19 +50,6 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
public int compare(ByteArray code1, ByteArray code2) {
return code1.compareTo(code2);
}
-
- @Override
- public void serialize(ByteArray code, ByteBuffer buffer) {
- if (code == null)
- BytesUtil.writeByteArray(null, 0, 0, buffer);
- else
- BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer);
- }
-
- @Override
- public ByteArray deserialize(ByteBuffer buffer) {
- return new ByteArray(BytesUtil.readByteArray(buffer));
- }
};
}
@@ -79,8 +64,8 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
}
@Override
- public IFilterCodeSystem<ByteArray> getFilterCodeSystem() {
- return filterCS;
+ public IGTComparator getComparator() {
+ return comparator;
}
// ============================================================================
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/10f075d6/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
index e8565ca..234d0c3 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
@@ -14,7 +14,6 @@ import java.util.Set;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.IFilterCodeSystem;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
@@ -36,11 +35,11 @@ public class GTScanRangePlanner {
public GTScanRangePlanner(GTInfo info) {
this.info = info;
- IFilterCodeSystem<ByteArray> cs = info.codeSystem.getFilterCodeSystem();
- this.byteUnknownIsSmaller = byteComparatorTreatsUnknownSmaller(cs);
- this.byteUnknownIsBigger = byteComparatorTreatsUnknownBigger(cs);
- this.recordUnknownIsSmaller = recordComparatorTreatsUnknownSmaller(cs);
- this.recordUnknownIsBigger = recordComparatorTreatsUnknownBigger(cs);
+ IGTComparator comp = info.codeSystem.getComparator();
+ this.byteUnknownIsSmaller = byteComparatorTreatsUnknownSmaller(comp);
+ this.byteUnknownIsBigger = byteComparatorTreatsUnknownBigger(comp);
+ this.recordUnknownIsSmaller = recordComparatorTreatsUnknownSmaller(comp);
+ this.recordUnknownIsBigger = recordComparatorTreatsUnknownBigger(comp);
}
// return empty list meaning filter is always false
@@ -325,7 +324,7 @@ public class GTScanRangePlanner {
if (equals != null) {
return equals.isEmpty();
} else if (begin.array() != null && end.array() != null) {
- return info.codeSystem.getFilterCodeSystem().compare(begin, end) > 0;
+ return info.codeSystem.getComparator().compare(begin, end) > 0;
} else {
return false;
}
@@ -425,7 +424,7 @@ public class GTScanRangePlanner {
}
}
- public static ComparatorEx<ByteArray> byteComparatorTreatsUnknownSmaller(final IFilterCodeSystem<ByteArray> cs) {
+ public static ComparatorEx<ByteArray> byteComparatorTreatsUnknownSmaller(final IGTComparator comp) {
return new ComparatorEx<ByteArray>() {
@Override
public int compare(ByteArray a, ByteArray b) {
@@ -434,12 +433,12 @@ public class GTScanRangePlanner {
else if (b.array() == null)
return 1;
else
- return cs.compare(a, b);
+ return comp.compare(a, b);
}
};
}
- public static ComparatorEx<ByteArray> byteComparatorTreatsUnknownBigger(final IFilterCodeSystem<ByteArray> cs) {
+ public static ComparatorEx<ByteArray> byteComparatorTreatsUnknownBigger(final IGTComparator comp) {
return new ComparatorEx<ByteArray>() {
@Override
public int compare(ByteArray a, ByteArray b) {
@@ -448,17 +447,17 @@ public class GTScanRangePlanner {
else if (b.array() == null)
return -1;
else
- return cs.compare(a, b);
+ return comp.compare(a, b);
}
};
}
- public static ComparatorEx<GTRecord> recordComparatorTreatsUnknownSmaller(IFilterCodeSystem<ByteArray> cs) {
- return new RecordComparator(byteComparatorTreatsUnknownSmaller(cs));
+ public static ComparatorEx<GTRecord> recordComparatorTreatsUnknownSmaller(IGTComparator comp) {
+ return new RecordComparator(byteComparatorTreatsUnknownSmaller(comp));
}
- public static ComparatorEx<GTRecord> recordComparatorTreatsUnknownBigger(IFilterCodeSystem<ByteArray> cs) {
- return new RecordComparator(byteComparatorTreatsUnknownBigger(cs));
+ public static ComparatorEx<GTRecord> recordComparatorTreatsUnknownBigger(IGTComparator comp) {
+ return new RecordComparator(byteComparatorTreatsUnknownBigger(comp));
}
private static class RecordComparator extends ComparatorEx<GTRecord> {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/10f075d6/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
index d4936c4..2532659 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTUtil.java
@@ -6,9 +6,11 @@ import java.util.List;
import java.util.Set;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.IFilterCodeSystem;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilterSerializer;
import org.apache.kylin.metadata.model.ColumnDesc;
@@ -45,6 +47,8 @@ public class GTUtil {
final List<TblColRef> colMapping, final boolean encodeConstants, //
final Set<TblColRef> unevaluatableColumnCollector) {
+ IFilterCodeSystem<ByteArray> filterCodeSystem = wrap(info.codeSystem.getComparator());
+
byte[] bytes = TupleFilterSerializer.serialize(rootFilter, new TupleFilterSerializer.Decorator() {
@Override
public TupleFilter onSerialize(TupleFilter filter) {
@@ -182,9 +186,36 @@ public class GTUtil {
return null;
}
}
- }, info.codeSystem.getFilterCodeSystem());
+ }, filterCodeSystem);
- return TupleFilterSerializer.deserialize(bytes, info.codeSystem.getFilterCodeSystem());
+ return TupleFilterSerializer.deserialize(bytes, filterCodeSystem);
}
+ public static IFilterCodeSystem<ByteArray> wrap(final IGTComparator comp) {
+ return new IFilterCodeSystem<ByteArray>() {
+
+ @Override
+ public int compare(ByteArray o1, ByteArray o2) {
+ return comp.compare(o1, o2);
+ }
+
+ @Override
+ public boolean isNull(ByteArray code) {
+ return comp.isNull(code);
+ }
+
+ @Override
+ public void serialize(ByteArray code, ByteBuffer buffer) {
+ if (code == null)
+ BytesUtil.writeByteArray(null, 0, 0, buffer);
+ else
+ BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer);
+ }
+
+ @Override
+ public ByteArray deserialize(ByteBuffer buffer) {
+ return new ByteArray(BytesUtil.readByteArray(buffer));
+ }
+ };
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/10f075d6/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTCodeSystem.java
index 368cc32..644c94d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTCodeSystem.java
@@ -2,16 +2,14 @@ package org.apache.kylin.storage.gridtable;
import java.nio.ByteBuffer;
-import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.filter.IFilterCodeSystem;
import org.apache.kylin.metadata.measure.MeasureAggregator;
public interface IGTCodeSystem {
void init(GTInfo info);
- IFilterCodeSystem<ByteArray> getFilterCodeSystem();
+ IGTComparator getComparator();
/** Return the length of code starting at the specified buffer, buffer position must not change after return */
int codeLength(int col, ByteBuffer buf);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/10f075d6/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTComparator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTComparator.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTComparator.java
new file mode 100644
index 0000000..442adcc
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTComparator.java
@@ -0,0 +1,15 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.util.Comparator;
+
+import org.apache.kylin.common.util.ByteArray;
+
+public interface IGTComparator extends Comparator<ByteArray> {
+
+ /** if given code represents the NULL value */
+ boolean isNull(ByteArray code);
+
+ /** compare two values by their codes */
+ // int compare(T code1, T code2);
+
+}