You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/14 01:01:24 UTC
[11/50] incubator-kylin git commit: KYLIN-625, initial skeleton
KYLIN-625, initial skeleton
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b093bd5e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b093bd5e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b093bd5e
Branch: refs/heads/streaming
Commit: b093bd5e2dd4d2c14b2d0462194f0cdafc923a7e
Parents: 0accec1
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Mar 6 18:16:11 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Mar 6 18:16:11 2015 +0800
----------------------------------------------------------------------
.../common/hll/HyperLogLogPlusCounter.java | 18 +++++
.../metadata/measure/BigDecimalSerializer.java | 18 ++++-
.../metadata/measure/DataTypeSerializer.java | 83 ++++++++++++++++++++
.../metadata/measure/DoubleSerializer.java | 7 +-
.../kylin/metadata/measure/HLLCSerializer.java | 7 +-
.../kylin/metadata/measure/LongSerializer.java | 14 +++-
.../kylin/metadata/measure/MeasureCodec.java | 8 +-
.../metadata/measure/MeasureSerializer.java | 68 ----------------
.../kylin/storage/gridtable/GTBuilder.java | 68 ++++++++++++++++
.../apache/kylin/storage/gridtable/GTInfo.java | 52 ++++++++++++
.../kylin/storage/gridtable/GTRecord.java | 15 ++++
.../kylin/storage/gridtable/GTRowBlock.java | 33 ++++++++
.../apache/kylin/storage/gridtable/GTStore.java | 24 ++++++
.../kylin/storage/gridtable/GridTable.java | 19 +++++
.../kylin/storage/gridtable/IKVStoreReader.java | 11 +++
.../kylin/storage/gridtable/IKVStoreWriter.java | 11 +++
16 files changed, 377 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
index 381dea6..686321b 100644
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
@@ -214,6 +214,24 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
in.get(registers);
}
}
+
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+ int len;
+
+ byte scheme = in.get();
+ if (scheme == 0) { // map scheme
+ int size = BytesUtil.readVInt(in);
+ int indexLen = getRegisterIndexSize();
+ len = in.position() - mark + (indexLen + 1) * size;
+ } else {
+ len = in.position() - mark + m;
+ }
+
+ in.position(mark);
+ return len;
+
+ }
public void writeRegistersArray(final ByteBuffer out) {
out.put(this.registers);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java
index 2eab9e4..78654ea 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSerializer.java
@@ -23,14 +23,13 @@ import java.math.BigInteger;
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.util.Bytes;
-
import org.apache.kylin.common.util.BytesUtil;
/**
* @author yangli9
*
*/
-public class BigDecimalSerializer extends MeasureSerializer<BigDecimal> {
+public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
@Override
public void serialize(BigDecimal value, ByteBuffer out) {
@@ -52,6 +51,20 @@ public class BigDecimalSerializer extends MeasureSerializer<BigDecimal> {
return new BigDecimal(new BigInteger(bytes), scale);
}
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+
+ @SuppressWarnings("unused")
+ int scale = BytesUtil.readVInt(in);
+ int n = BytesUtil.readVInt(in);
+ int len = in.position() - mark + n;
+
+ in.position(mark);
+ return len;
+ }
+
@Override
public BigDecimal valueOf(byte[] value) {
if (value == null)
@@ -59,5 +72,4 @@ public class BigDecimalSerializer extends MeasureSerializer<BigDecimal> {
else
return new BigDecimal(Bytes.toString(value));
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/metadata/src/main/java/org/apache/kylin/metadata/measure/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/DataTypeSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/DataTypeSerializer.java
new file mode 100644
index 0000000..ecaaea7
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/DataTypeSerializer.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.measure;
+
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+
+import org.apache.kylin.common.util.BytesSerializer;
+import org.apache.kylin.metadata.model.DataType;
+
+/**
+ * @author yangli9
+ *
+ */
+abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
+
+ final static HashMap<String, Class<?>> implementations = new HashMap<String, Class<?>>();
+ static {
+ implementations.put("decimal", BigDecimalSerializer.class);
+ implementations.put("double", DoubleSerializer.class);
+ implementations.put("float", DoubleSerializer.class);
+ implementations.put("bigint", LongSerializer.class);
+ implementations.put("long", LongSerializer.class);
+ implementations.put("integer", LongSerializer.class);
+ implementations.put("int", LongSerializer.class);
+ }
+
+ public static DataTypeSerializer<?> create(String dataType) {
+ DataType type = DataType.getInstance(dataType);
+ if (type.isHLLC()) {
+ return new HLLCSerializer(type.getPrecision());
+ }
+
+ Class<?> clz = implementations.get(type.getName());
+ if (clz == null)
+ throw new RuntimeException("No MeasureSerializer for type " + dataType);
+
+ try {
+ return (DataTypeSerializer<?>) clz.newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e); // never happen
+ }
+ }
+
+ /** peek into buffer and return the length of serialization */
+ abstract public int peekLength(ByteBuffer in);
+
+ /** convert from String to obj */
+ abstract public T valueOf(byte[] value);
+
+ public T valueOf(String value) {
+ try {
+ return valueOf(value.getBytes("UTF-8"));
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e); // never happen
+ }
+ }
+
+ /** convert from obj to string */
+ public String toString(T value) {
+ if (value == null)
+ return "NULL";
+ else
+ return value.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSerializer.java
index ea2678c..2874a57 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSerializer.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.io.DoubleWritable;
* @author yangli9
*
*/
-public class DoubleSerializer extends MeasureSerializer<DoubleWritable> {
+public class DoubleSerializer extends DataTypeSerializer<DoubleWritable> {
// avoid mass object creation
DoubleWritable current = new DoubleWritable();
@@ -44,6 +44,11 @@ public class DoubleSerializer extends MeasureSerializer<DoubleWritable> {
}
@Override
+ public int peekLength(ByteBuffer in) {
+ return 8;
+ }
+
+ @Override
public DoubleWritable valueOf(byte[] value) {
if (value == null)
current.set(0d);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
index e3fcbee..2303c82 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
@@ -27,7 +27,7 @@ import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
* @author yangli9
*
*/
-public class HLLCSerializer extends MeasureSerializer<HyperLogLogPlusCounter> {
+public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounter> {
HyperLogLogPlusCounter current;
@@ -55,6 +55,11 @@ public class HLLCSerializer extends MeasureSerializer<HyperLogLogPlusCounter> {
}
@Override
+ public int peekLength(ByteBuffer in) {
+ return current.peekLength(in);
+ }
+
+ @Override
public HyperLogLogPlusCounter valueOf(byte[] value) {
current.clear();
if (value == null)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSerializer.java
index 90c1ab9..9dca987 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/LongSerializer.java
@@ -22,14 +22,13 @@ import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
-
import org.apache.kylin.common.util.BytesUtil;
/**
* @author yangli9
*
*/
-public class LongSerializer extends MeasureSerializer<LongWritable> {
+public class LongSerializer extends DataTypeSerializer<LongWritable> {
// avoid mass object creation
LongWritable current = new LongWritable();
@@ -46,6 +45,17 @@ public class LongSerializer extends MeasureSerializer<LongWritable> {
}
@Override
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+
+ BytesUtil.readVLong(in);
+ int len = in.position() - mark;
+
+ in.position(mark);
+ return len;
+ }
+
+ @Override
public LongWritable valueOf(byte[] value) {
if (value == null)
current.set(0L);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
index 6fd928f..95a246c 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
@@ -33,7 +33,7 @@ import org.apache.kylin.metadata.model.MeasureDesc;
public class MeasureCodec {
int nMeasures;
- MeasureSerializer[] serializers;
+ DataTypeSerializer[] serializers;
public MeasureCodec(Collection<MeasureDesc> measureDescs) {
this((MeasureDesc[]) measureDescs.toArray(new MeasureDesc[measureDescs.size()]));
@@ -53,14 +53,14 @@ public class MeasureCodec {
private void init(String[] dataTypes) {
nMeasures = dataTypes.length;
- serializers = new MeasureSerializer[nMeasures];
+ serializers = new DataTypeSerializer[nMeasures];
for (int i = 0; i < nMeasures; i++) {
- serializers[i] = MeasureSerializer.create(dataTypes[i]);
+ serializers[i] = DataTypeSerializer.create(dataTypes[i]);
}
}
- public MeasureSerializer getSerializer(int idx) {
+ public DataTypeSerializer getSerializer(int idx) {
return serializers[idx];
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureSerializer.java
deleted file mode 100644
index e245bc2..0000000
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureSerializer.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.util.HashMap;
-
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.metadata.model.DataType;
-
-/**
- * @author yangli9
- *
- */
-abstract public class MeasureSerializer<T> implements BytesSerializer<T> {
-
- final static HashMap<String, Class<?>> implementations = new HashMap<String, Class<?>>();
- static {
- implementations.put("decimal", BigDecimalSerializer.class);
- implementations.put("double", DoubleSerializer.class);
- implementations.put("float", DoubleSerializer.class);
- implementations.put("bigint", LongSerializer.class);
- implementations.put("long", LongSerializer.class);
- implementations.put("integer", LongSerializer.class);
- implementations.put("int", LongSerializer.class);
- }
-
- public static MeasureSerializer<?> create(String dataType) {
- DataType type = DataType.getInstance(dataType);
- if (type.isHLLC()) {
- return new HLLCSerializer(type.getPrecision());
- }
-
- Class<?> clz = implementations.get(type.getName());
- if (clz == null)
- throw new RuntimeException("No MeasureSerializer for type " + dataType);
-
- try {
- return (MeasureSerializer<?>) clz.newInstance();
- } catch (Exception e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- abstract public T valueOf(byte[] value);
-
- public String toString(T value) {
- if (value == null)
- return "NULL";
- else
- return value.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
new file mode 100644
index 0000000..9e29b52
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTBuilder.java
@@ -0,0 +1,68 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.io.Closeable;
+import java.io.Flushable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.storage.gridtable.GTStore.GTBlockWriter;
+
+public class GTBuilder implements Closeable, Flushable {
+
+ final private GTInfo info;
+ final private GTBlockWriter writer;
+
+ private GTRowBlock block;
+
+ public GTBuilder(GTInfo info, int shard, GTStore store) {
+ this.info = info;
+ this.writer = store.rebuild(shard);
+ this.block = new GTRowBlock(info);
+ }
+
+ public void write(GTRecord r) throws IOException {
+ // add record to block
+ if (block.isEmpty()) {
+ makePrimaryKey(r, block.primaryKey);
+ }
+ for (int c = 0; c < info.nColBlocks; c++) {
+ ByteBuffer cellBuf = block.cellBlocks[c];
+ for (int i = info.colBlockCuts[c], end = info.colBlockCuts[c + 1]; i < end; i++) {
+ append(cellBuf, r.cols[i]);
+ }
+ }
+
+ block.nRows++;
+ if (block.nRows >= info.rowBlockSize || info.isRowBlockEnabled() == false) {
+ flush();
+ }
+ }
+
+ private void makePrimaryKey(GTRecord r, ByteBuffer buf) {
+ buf.clear();
+
+ for (int i : info.primaryKey) {
+ append(buf, r.cols[i]);
+ }
+
+ buf.flip();
+ }
+
+ private void append(ByteBuffer buf, ByteBuffer data) {
+ buf.put(data.array(), data.arrayOffset(), data.limit());
+ }
+
+ @Override
+ public void flush() throws IOException {
+ writer.write(block);
+ block.clear();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (block.isEmpty() == false) {
+ flush();
+ }
+ writer.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
new file mode 100644
index 0000000..11a3083
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
@@ -0,0 +1,52 @@
+package org.apache.kylin.storage.gridtable;
+
+import org.apache.kylin.metadata.model.DataType;
+
+public class GTInfo {
+
+ // column schema
+ int nColumns;
+ DataType[] colType;
+ boolean[] colIsMetrics;
+ int maxRecordLength; // column length can vary
+
+ // grid info
+ int[] primaryKey; // columns sorted and unique
+ int rowBlockSize; // 0: no row block
+ int nColBlocks;
+ int[] colBlockCuts; // [i]: the start index of i.th block; [i+1]: the exclusive end index of i.th block
+
+ // sharding & rowkey
+ int nShards; // 0: no sharding
+ byte[] rowkeyPrefix;
+
+ public boolean isShardingEnabled() {
+ return nShards > 0;
+ }
+
+ public boolean isRowBlockEnabled() {
+ return rowBlockSize > 0;
+ }
+
+ void validate() {
+ if (nColBlocks != colBlockCuts.length - 1) {
+ throw new IllegalStateException();
+ }
+
+ for (int i = 0; i < nColBlocks; i++) {
+ int cbStart = colBlockCuts[i], cbEnd = colBlockCuts[i + 1];
+ if (cbStart >= cbEnd)
+ throw new IllegalStateException();
+
+ // when row block disabled, primary key is persisted on rowkey, thus shall not repeat in column block
+ if (isRowBlockEnabled() == false) {
+ for (int ii : primaryKey) {
+ if (ii >= cbStart && ii < cbEnd)
+ throw new IllegalStateException();
+ }
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
new file mode 100644
index 0000000..7943965
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
@@ -0,0 +1,15 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.nio.ByteBuffer;
+
+public class GTRecord {
+
+ final GTInfo info;
+ final ByteBuffer[] cols;
+
+ public GTRecord(GTInfo info) {
+ this.info = info;
+ this.cols = new ByteBuffer[info.nColumns];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
new file mode 100644
index 0000000..4526ef4
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
@@ -0,0 +1,33 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.nio.ByteBuffer;
+
+class GTRowBlock {
+
+ private static final int PRIMARY_KEY_CAPACITY = 2048;
+ private static final int CELL_BLOCK_CAPACITY = 128 * 1024;
+
+ int nRows;
+ ByteBuffer primaryKey; // the primary key of the first row
+ ByteBuffer[] cellBlocks; // cells for each column block
+
+ public GTRowBlock(GTInfo info) {
+ primaryKey = ByteBuffer.allocate(PRIMARY_KEY_CAPACITY);
+ cellBlocks = new ByteBuffer[info.nColBlocks];
+ for (int i = 0; i < cellBlocks.length; i++) {
+ cellBlocks[i] = ByteBuffer.allocate(CELL_BLOCK_CAPACITY);
+ }
+ }
+
+ public boolean isEmpty() {
+ return nRows == 0;
+ }
+
+ public void clear() {
+ nRows = 0;
+ primaryKey.clear();
+ for (int i = 0; i < cellBlocks.length; i++) {
+ cellBlocks[i].clear();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
new file mode 100644
index 0000000..3ccaa00
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTStore.java
@@ -0,0 +1,24 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+
+public interface GTStore {
+
+ public GTInfo getInfo();
+
+ public String getStorageDescription();
+
+ public GTBlockWriter rebuild(int shard);
+
+ public GTBlockScanner scan(ByteBuffer pkStart, ByteBuffer pkEndExclusive, int[] colBlocks);
+
+ public interface GTBlockWriter extends Closeable {
+ void write(GTRowBlock block) throws IOException;
+ }
+
+ public interface GTBlockScanner extends Iterator<GTRowBlock>, Closeable {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
new file mode 100644
index 0000000..7eb8657
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
@@ -0,0 +1,19 @@
+package org.apache.kylin.storage.gridtable;
+
+public class GridTable {
+
+ final GTInfo info;
+
+ public GridTable(GTInfo info, GTStore store) {
+ this.info = info;
+ }
+
+ public GTBuilder rebuild(GTStore store) {
+ assert info.nShards == 0;
+ return rebuild(store, 0);
+ }
+
+ public GTBuilder rebuild(GTStore store, int shard) {
+ return new GTBuilder(info, shard, store);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java
new file mode 100644
index 0000000..d62d478
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreReader.java
@@ -0,0 +1,11 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.io.Closeable;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+
+import org.apache.hadoop.hbase.util.Pair;
+
+public interface IKVStoreReader extends Iterator<Pair<ByteBuffer, ByteBuffer>>, Closeable {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b093bd5e/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java
new file mode 100644
index 0000000..8eeba95
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IKVStoreWriter.java
@@ -0,0 +1,11 @@
+package org.apache.kylin.storage.gridtable;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public interface IKVStoreWriter extends Closeable {
+
+ void write(ByteBuffer key, ByteBuffer value) throws IOException;
+
+}