You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/14 01:01:22 UTC
[47/50] [abbrv] incubator-kylin git commit: KYLIN-625, draft impl,
ready for very first test
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
new file mode 100644
index 0000000..a80f4d9
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
@@ -0,0 +1,169 @@
+package org.apache.kylin.storage.gridtable;
+
+import it.uniroma3.mat.extendedset.intset.ConciseSet;
+
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.metadata.filter.IFilterCodeSystem;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.apache.kylin.storage.gridtable.IGTStore.IGTStoreScanner;
+
+class GTRawScanner implements IGTScanner {
+
+ final GTInfo info;
+ final IGTStoreScanner storeScanner;
+ final TupleFilter filter;
+ final BitSet selectedColBlocks;
+
+ private GTRowBlock currentBlock;
+ private int currentRow;
+ private GTRecord next;
+ final private GTRecord oneRecord; // avoid instance creation
+ final private TupleAdapter oneTuple; // avoid instance creation
+
+ GTRawScanner(GTInfo info, IGTStore store, GTRecord pkStart, GTRecord pkEndExclusive, BitSet dimensions, BitSet metrics, TupleFilter filter) {
+ this.info = info;
+ this.filter = filter;
+
+ ByteArray start = pkStart.exportColumns(info.primaryKey);
+ ByteArray endEx = pkEndExclusive.exportColumns(info.primaryKey);
+
+ ConciseSet selectedRowBlocks = computeHitRowBlocks(start, endEx, filter);
+ this.selectedColBlocks = computeHitColumnBlocks(dimensions, metrics);
+
+ this.storeScanner = store.scan(start, endEx, selectedRowBlocks, selectedColBlocks);
+ this.oneRecord = new GTRecord(info);
+ this.oneTuple = new TupleAdapter(oneRecord);
+ }
+
+ private BitSet computeHitColumnBlocks(BitSet dimensions, BitSet metrics) {
+ BitSet result = new BitSet();
+ for (int i = 0; i < info.colBlocks.length; i++) {
+ BitSet cb = info.colBlocks[i];
+ if (cb.intersects(dimensions) || cb.intersects(metrics)) {
+ result.set(i);
+ }
+ }
+ return result;
+ }
+
+ private ConciseSet computeHitRowBlocks(ByteArray start, ByteArray endEx, TupleFilter filter) {
+ // TODO block level index
+ return null;
+ }
+
+ @Override
+ public void close() throws IOException {
+ storeScanner.close();
+ }
+
+ @Override
+ public Iterator<GTRecord> iterator() {
+ return new Iterator<GTRecord>() {
+
+ @Override
+ public boolean hasNext() {
+ if (next != null)
+ return true;
+
+ IFilterCodeSystem<ByteArray> filterCodeSystem = info.codeSystem.getFilterCodeSystem();
+
+ while (fetchNext()) {
+ if (filter != null && filter.evaluate(oneTuple, filterCodeSystem) == false) {
+ continue;
+ }
+ next = oneRecord;
+ return true;
+ }
+ return false;
+ }
+
+ private boolean fetchNext() {
+ if (info.isRowBlockEnabled()) {
+ return fetchNextRowBlockEnabled();
+ } else {
+ return fetchNextRowBlockDisabled();
+ }
+ }
+
+ private boolean fetchNextRowBlockDisabled() {
+ // row block disabled, every block is one row
+ if (storeScanner.hasNext() == false)
+ return false;
+
+ // when row block disabled, PK is persisted in block primary key (not in cell block)
+ currentBlock = storeScanner.next();
+ oneRecord.loadPrimaryKey(currentBlock.primaryKeyBuffer);
+ for (int c = selectedColBlocks.nextSetBit(0); c >= 0; c = selectedColBlocks.nextSetBit(c + 1)) {
+ oneRecord.loadCellBlock(c, currentBlock.cellBlockBuffers[c]);
+ }
+ return true;
+ }
+
+ private boolean fetchNextRowBlockEnabled() {
+ while (true) {
+ // get a block
+ if (currentBlock == null) {
+ if (storeScanner.hasNext()) {
+ currentBlock = storeScanner.next();
+ currentRow = 0;
+ } else {
+ return false;
+ }
+ }
+ // if block exhausted, try next block
+ if (currentRow >= currentBlock.nRows) {
+ currentBlock = null;
+ continue;
+ }
+ // fetch a row
+ for (int c = selectedColBlocks.nextSetBit(0); c >= 0; c = selectedColBlocks.nextSetBit(c + 1)) {
+ oneRecord.loadCellBlock(c, currentBlock.cellBlockBuffers[c]);
+ }
+ return true;
+ }
+ }
+
+ @Override
+ public GTRecord next() {
+ // fetch next record
+ if (next == null) {
+ hasNext();
+ if (next == null)
+ throw new NoSuchElementException();
+ }
+
+ GTRecord result = next;
+ next = null;
+ return result;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+
+ private static class TupleAdapter implements IEvaluatableTuple {
+
+ private GTRecord r;
+
+ private TupleAdapter(GTRecord r) {
+ this.r = r;
+ }
+
+ @Override
+ public Object getValue(TblColRef col) {
+ return r.cols[col.getColumn().getZeroBasedIndex()];
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
index 7943965..146d4c7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
@@ -1,15 +1,140 @@
package org.apache.kylin.storage.gridtable;
import java.nio.ByteBuffer;
+import java.util.BitSet;
-public class GTRecord {
+import org.apache.kylin.common.util.ByteArray;
+
+public class GTRecord implements Comparable<GTRecord> {
final GTInfo info;
- final ByteBuffer[] cols;
+ final ByteArray[] cols;
+
+ BitSet maskForEqualHashComp;
public GTRecord(GTInfo info) {
this.info = info;
- this.cols = new ByteBuffer[info.nColumns];
+ this.cols = new ByteArray[info.nColumns];
+ for (int i = 0; i < cols.length; i++)
+ this.cols[i] = new ByteArray();
+ this.maskForEqualHashComp = info.colAll;
+ }
+
+ public GTRecord copy() {
+ return copy(info.colAll);
+ }
+
+ public GTRecord copy(BitSet selectedCols) {
+ int len = 0;
+ for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
+ len += cols[i].length();
+ }
+
+ byte[] space = new byte[len];
+
+ GTRecord copy = new GTRecord(info);
+ copy.maskForEqualHashComp = this.maskForEqualHashComp;
+ int pos = 0;
+ for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
+ System.arraycopy(cols[i].array(), cols[i].offset(), space, pos, cols[i].length());
+ copy.cols[i].set(space, pos, cols[i].length());
+ pos += cols[i].length();
+ }
+
+ return copy;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+
+ GTRecord o = (GTRecord) obj;
+ if (this.info != o.info)
+ return false;
+ if (this.maskForEqualHashComp != o.maskForEqualHashComp)
+ return false;
+ for (int i = maskForEqualHashComp.nextSetBit(0); i >= 0; i = maskForEqualHashComp.nextSetBit(i + 1)) {
+ if (this.cols[i].equals(o.cols[i]) == false)
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 1;
+ for (int i = maskForEqualHashComp.nextSetBit(0); i >= 0; i = maskForEqualHashComp.nextSetBit(i + 1)) {
+ hash = (31 * hash) + cols[i].hashCode();
+ }
+ return hash;
+ }
+
+ @Override
+ public int compareTo(GTRecord o) {
+ int comp = 0;
+ for (int i = maskForEqualHashComp.nextSetBit(0); i >= 0; i = maskForEqualHashComp.nextSetBit(i + 1)) {
+ comp = this.cols[i].compareTo(o.cols[i]);
+ if (comp != 0)
+ return comp;
+ }
+ return comp;
+ }
+
+
+ // ============================================================================
+
+ ByteArray exportColumns(BitSet selectedCols) {
+ int len = 0;
+ for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
+ len += cols[i].length();
+ }
+
+ ByteArray buf = ByteArray.allocate(len);
+ exportColumns(info.primaryKey, buf);
+ return buf;
+ }
+
+ /** write data to given buffer, like serialize */
+ void exportColumns(BitSet selectedCols, ByteArray buf) {
+ int pos = 0;
+ for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
+ System.arraycopy(cols[i].array(), cols[i].offset(), buf.array(), buf.offset() + pos, cols[i].length());
+ pos += cols[i].length();
+ }
+ buf.setLength(pos);
+ }
+
+ /** write data to given buffer, like serialize */
+ void exportColumnBlock(int c, ByteBuffer buf) {
+ BitSet setselectedCols = info.colBlocks[c];
+ for (int i = setselectedCols.nextSetBit(0); i >= 0; i = setselectedCols.nextSetBit(i + 1)) {
+ buf.put(cols[i].array(), cols[i].offset(), cols[i].length());
+ }
+ }
+
+ /** change pointers to point to data in given buffer, UNLIKE deserialize */
+ void loadPrimaryKey(ByteBuffer buf) {
+ loadColumns(info.primaryKey, buf);
+ }
+
+ /** change pointers to point to data in given buffer, UNLIKE deserialize */
+ void loadCellBlock(int c, ByteBuffer buf) {
+ loadColumns(info.colBlocks[c], buf);
+ }
+
+ /** change pointers to point to data in given buffer, UNLIKE deserialize */
+ void loadColumns(BitSet selectedCols, ByteBuffer buf) {
+ int pos = buf.position();
+ for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
+ int len = info.codeSystem.codeLength(i, buf);
+ cols[i].set(buf.array(), buf.arrayOffset() + pos, len);
+ buf.position(pos + len);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
index 6db08ab..ca4a35d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
@@ -2,33 +2,114 @@ package org.apache.kylin.storage.gridtable;
import java.nio.ByteBuffer;
-class GTRowBlock {
+import org.apache.kylin.common.util.ByteArray;
- private static final int PRIMARY_KEY_CAPACITY = 2048;
- private static final int CELL_BLOCK_CAPACITY = 128 * 1024;
+public class GTRowBlock {
+ /** create a row block, allocate memory, get ready for incoming data */
+ public static GTRowBlock allocate(GTInfo info) {
+ GTRowBlock b = new GTRowBlock(info);
+
+ byte[] array = new byte[info.maxRecordLength];
+ b.primaryKey.set(array);
+ b.primaryKeyBuffer = ByteBuffer.wrap(array);
+
+ int maxRows = info.isRowBlockEnabled() ? info.rowBlockSize : 1;
+ for (int i = 0; i < b.cellBlocks.length; i++) {
+ array = new byte[info.maxRecordLength * maxRows];
+ b.cellBlocks[i].set(array);
+ b.cellBlockBuffers[i] = ByteBuffer.wrap(array);
+ }
+ return b;
+ }
+
+ final GTInfo info;
+
int seqId; // 0, 1, 2...
int nRows;
- ByteBuffer primaryKey; // the primary key of the first row
- ByteBuffer[] cellBlocks; // cells for each column block
+ ByteArray primaryKey; // the primary key of the first row
+ ByteBuffer primaryKeyBuffer;
+ ByteArray[] cellBlocks; // cells for each column block
+ ByteBuffer[] cellBlockBuffers;
+ /** create a row block that has no underlying memory */
public GTRowBlock(GTInfo info) {
- primaryKey = ByteBuffer.allocate(PRIMARY_KEY_CAPACITY);
- cellBlocks = new ByteBuffer[info.nColBlocks];
+ this.info = info;
+ primaryKey = new ByteArray();
+ cellBlocks = new ByteArray[info.colBlocks.length];
for (int i = 0; i < cellBlocks.length; i++) {
- cellBlocks[i] = ByteBuffer.allocate(CELL_BLOCK_CAPACITY);
+ cellBlocks[i] = new ByteArray();
}
+ cellBlockBuffers = new ByteBuffer[info.colBlocks.length];
+ }
+
+ public GTRowBlock copy() {
+ GTRowBlock copy = new GTRowBlock(info);
+
+ ByteBuffer buf = ByteBuffer.allocate(this.exportLength());
+ this.export(buf);
+ buf.clear();
+ copy.load(buf);
+
+ return copy;
}
+ public void rewindBuffers() {
+ primaryKeyBuffer.rewind();
+ for (int i = 0; i < cellBlockBuffers.length; i++) {
+ cellBlockBuffers[i].rewind();
+ }
+ }
+
public boolean isEmpty() {
return nRows == 0;
}
public void clear() {
nRows = 0;
- primaryKey.clear();
- for (int i = 0; i < cellBlocks.length; i++) {
- cellBlocks[i].clear();
+ rewindBuffers();
+ }
+
+ public int exportLength() {
+ int len = 4 + 4 + (4 + primaryKey.length());
+ for (ByteArray array : cellBlocks) {
+ len += 4 + array.length();
+ }
+ return len;
+ }
+
+ /** write data to given buffer, like serialize */
+ public void export(ByteBuffer buf) {
+ buf.putInt(seqId);
+ buf.putInt(nRows);
+ export(primaryKey, buf);
+ for (ByteArray cb : cellBlocks) {
+ export(cb, buf);
}
}
+
+ private void export(ByteArray array, ByteBuffer buf) {
+ buf.putInt(array.length());
+ buf.put(array.array(), array.offset(), array.length());
+ }
+
+ /** change pointers to point to data in given buffer, UNLIKE deserialize */
+ public void load(ByteBuffer buf) {
+ seqId = buf.getInt();
+ nRows = buf.getInt();
+ load(primaryKey, buf);
+ primaryKeyBuffer = ByteBuffer.wrap(primaryKey.array(), primaryKey.offset(), primaryKey.length());
+ for (int i = 0; i < info.colBlocks.length; i++) {
+ ByteArray cb = cellBlocks[i];
+ load(cb, buf);
+ cellBlockBuffers[i] = ByteBuffer.wrap(cb.array(), cb.offset(), cb.length());
+ }
+ }
+
+ private void load(ByteArray array, ByteBuffer buf) {
+ int len = buf.getInt();
+ int pos = buf.position();
+ array.set(buf.array(), buf.arrayOffset() + pos, len);
+ buf.position(pos + len);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
deleted file mode 100644
index b665fd6..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package org.apache.kylin.storage.gridtable;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-
-public interface GTStore {
-
- public GTInfo getInfo();
-
- public String getStorageDescription();
-
- // ============================================================================
-
- public GTWriter rebuild(int shard);
-
- public GTScanner scan(ByteBuffer pkStart, ByteBuffer pkEndExclusive, ConciseSet selectedRowBlcoks, int[] selectedColBlocks);
-
- public interface GTWriter extends Closeable {
- void write(GTRowBlock block) throws IOException;
- }
-
- public interface GTScanner extends Iterator<GTRowBlock>, Closeable {
- }
-
- // ============================================================================
-
- public void saveRowBlockIndex(int col, GTRowBlockIndex index);
-
- public GTRowBlockIndex loadRowBlockIndex(int col);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
index 7eb8657..5d09928 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
@@ -1,19 +1,34 @@
package org.apache.kylin.storage.gridtable;
+import java.util.BitSet;
+
+import org.apache.kylin.metadata.filter.TupleFilter;
+
public class GridTable {
final GTInfo info;
+ final IGTStore store;
- public GridTable(GTInfo info, GTStore store) {
+ public GridTable(GTInfo info, IGTStore store) {
this.info = info;
+ this.store = store;
}
- public GTBuilder rebuild(GTStore store) {
- assert info.nShards == 0;
- return rebuild(store, 0);
+ public GTBuilder rebuild() {
+ assert info.isShardingEnabled() == false;
+ return rebuild(-1);
}
- public GTBuilder rebuild(GTStore store, int shard) {
+ public GTBuilder rebuild(int shard) {
+ assert shard < info.nShards;
return new GTBuilder(info, shard, store);
}
+
+ public IGTScanner scan(GTRecord pkStart, GTRecord pkEndExclusive, BitSet dimensions, BitSet metrics, TupleFilter filter) {
+ return new GTRawScanner(info, store, pkStart, pkEndExclusive, dimensions, metrics, filter);
+ }
+
+ public IGTScanner scanAndAggregate(GTRecord pkStart, GTRecord pkEndExclusive, BitSet dimensions, BitSet metrics, TupleFilter filter) {
+ return new GTAggregateScanner(info, store, pkStart, pkEndExclusive, dimensions, metrics, filter);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTCodeSystem.java
new file mode 100644
index 0000000..2e960a0
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTCodeSystem.java
@@ -0,0 +1,25 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.metadata.filter.IFilterCodeSystem;
+import org.apache.kylin.metadata.measure.MeasureAggregator;
+
+public interface IGTCodeSystem {
+
+ IFilterCodeSystem<ByteArray> getFilterCodeSystem();
+
+ /** return the length of code starting at the specified buffer, buffer position must not change after return */
+ int codeLength(int col, ByteBuffer buf);
+
+ /** encode a value into code */
+ void encodeColumnValue(int col, Object value, ByteBuffer buf);
+
+ /** decode a code into value */
+ Object decodeColumnValue(int col, ByteBuffer buf);
+
+ /** return an aggregator for metrics */
+ MeasureAggregator<?> newMetricsAggregator(int col);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
new file mode 100644
index 0000000..c51c301
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
@@ -0,0 +1,7 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.io.Closeable;
+
+public interface IGTScanner extends Iterable<GTRecord>, Closeable {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
new file mode 100644
index 0000000..621b66b
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
@@ -0,0 +1,37 @@
+package org.apache.kylin.storage.gridtable;
+
+import it.uniroma3.mat.extendedset.intset.ConciseSet;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.Iterator;
+
+import org.apache.kylin.common.util.ByteArray;
+
+public interface IGTStore {
+
+ public GTInfo getInfo();
+
+ public String getStorageDescription();
+
+ // ============================================================================
+
+ public IGTStoreWriter rebuild(int shard);
+
+ public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEndExclusive, ConciseSet selectedRowBlocks, BitSet selectedColBlocks);
+
+ public interface IGTStoreWriter extends Closeable {
+ void write(GTRowBlock block) throws IOException;
+ }
+
+ public interface IGTStoreScanner extends Iterator<GTRowBlock>, Closeable {
+ }
+
+ // ============================================================================
+
+ public void saveRowBlockIndex(int col, GTRowBlockIndex index);
+
+ public GTRowBlockIndex loadRowBlockIndex(int col);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java
deleted file mode 100644
index d62d478..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.apache.kylin.storage.gridtable;
-
-import java.io.Closeable;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-
-import org.apache.hadoop.hbase.util.Pair;
-
-public interface IKVStoreReader extends Iterator<Pair<ByteBuffer, ByteBuffer>>, Closeable {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java
deleted file mode 100644
index 8eeba95..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.apache.kylin.storage.gridtable;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public interface IKVStoreWriter extends Closeable {
-
- void write(ByteBuffer key, ByteBuffer value) throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
new file mode 100644
index 0000000..f342d7b
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
@@ -0,0 +1,96 @@
+package org.apache.kylin.storage.gridtable.memstore;
+
+import it.uniroma3.mat.extendedset.intset.ConciseSet;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.storage.gridtable.GTInfo;
+import org.apache.kylin.storage.gridtable.GTRowBlock;
+import org.apache.kylin.storage.gridtable.GTRowBlockIndex;
+import org.apache.kylin.storage.gridtable.IGTStore;
+
+public class GTSimpleMemStore implements IGTStore {
+
+ final GTInfo info;
+ final List<GTRowBlock> rowBlockList;
+
+ public GTSimpleMemStore(GTInfo info) {
+ this.info = info;
+ this.rowBlockList = new ArrayList<GTRowBlock>();
+
+ if (info.isShardingEnabled())
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public GTInfo getInfo() {
+ return info;
+ }
+
+ @Override
+ public String getStorageDescription() {
+ return this.toString();
+ }
+
+ @Override
+ public IGTStoreWriter rebuild(int shard) {
+
+ rowBlockList.clear();
+
+ return new IGTStoreWriter() {
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public void write(GTRowBlock block) throws IOException {
+ rowBlockList.add(block.copy());
+ }
+ };
+ }
+
+ @Override
+ public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEndExclusive, ConciseSet selectedRowBlocks, BitSet selectedColBlocks) {
+
+ return new IGTStoreScanner() {
+ Iterator<GTRowBlock> it = rowBlockList.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return it.hasNext();
+ }
+
+ @Override
+ public GTRowBlock next() {
+ GTRowBlock block = it.next();
+ block.rewindBuffers();
+ return block;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+ };
+ }
+
+ @Override
+ public void saveRowBlockIndex(int col, GTRowBlockIndex index) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public GTRowBlockIndex loadRowBlockIndex(int col) {
+ throw new UnsupportedOperationException();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
index cada02b..bbe7c36 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
@@ -4,9 +4,14 @@ import java.nio.ByteBuffer;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.metadata.tuple.ICodeSystem;
+import org.apache.kylin.metadata.filter.IFilterCodeSystem;
-public class DictCodeSystem implements ICodeSystem<String> {
+/**
+ * A simple code system where all values are dictionary IDs (fixed length bytes) encoded as ISO-8859-1 strings.
+ *
+ * @author yangli9
+ */
+public class DictCodeSystem implements IFilterCodeSystem<String> {
public static final DictCodeSystem INSTANCE = new DictCodeSystem();
@@ -15,16 +20,6 @@ public class DictCodeSystem implements ICodeSystem<String> {
}
@Override
- public String encode(Object value) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Object decode(String code) {
- throw new UnsupportedOperationException();
- }
-
- @Override
public boolean isNull(String value) {
if (value == null)
return true;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e290fb0/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
index fe9d336..95c6555 100644
--- a/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/filter/FilterBaseTest.java
@@ -27,7 +27,6 @@ import org.apache.kylin.metadata.filter.*;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ICodeSystem;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.storage.tuple.Tuple;
import org.apache.kylin.storage.tuple.TupleInfo;
@@ -38,7 +37,7 @@ import org.apache.kylin.storage.tuple.TupleInfo;
*/
public class FilterBaseTest {
- static final ICodeSystem CS = StringCodeSystem.INSTANCE;
+ static final IFilterCodeSystem CS = StringCodeSystem.INSTANCE;
protected List<TblColRef> buildGroups() {
List<TblColRef> groups = new ArrayList<TblColRef>();