You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/08/13 09:05:23 UTC
[03/10] incubator-kylin git commit: KYLIN-909 Adapt hbase cube to
GTStore
KYLIN-909 Adapt hbase cube to GTStore
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/dad5e8ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/dad5e8ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/dad5e8ac
Branch: refs/heads/KYLIN-909
Commit: dad5e8ac3375c896e2a7049b71263e733660b239
Parents: aebd191
Author: honma <ho...@ebay.com>
Authored: Tue Jul 28 09:48:18 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Aug 5 11:06:59 2015 +0800
----------------------------------------------------------------------
.../kylin/cube/gridtable/CubeCodeSystem.java | 84 +++----
.../cube/inmemcubing/ConcurrentDiskStore.java | 26 +--
.../kylin/cube/inmemcubing/MemDiskStore.java | 27 +--
.../kylin/gridtable/DefaultGTComparator.java | 22 ++
.../kylin/gridtable/GTAggregateScanner.java | 16 +-
.../java/org/apache/kylin/gridtable/GTInfo.java | 9 +-
.../apache/kylin/gridtable/GTRawScanner.java | 10 +-
.../kylin/gridtable/GTSampleCodeSystem.java | 27 +--
.../apache/kylin/gridtable/GTScanRequest.java | 33 ++-
.../org/apache/kylin/gridtable/GridTable.java | 9 +-
.../org/apache/kylin/gridtable/IGTStore.java | 14 +-
.../gridtable/memstore/GTSimpleMemStore.java | 14 +-
.../apache/kylin/dict/DictionaryManager.java | 15 +-
.../org/apache/kylin/storage/tuple/Tuple.java | 10 +-
.../src/main/resources/applicationContext.xml | 222 +++++++++----------
.../storage/cube/CubeHBaseEndpointRPC.java | 126 +++++++++++
.../apache/kylin/storage/cube/CubeHBaseRPC.java | 95 ++++++++
.../storage/cube/CubeHBaseReadonlyStore.java | 177 ---------------
.../kylin/storage/cube/CubeHBaseScanRPC.java | 106 +++++++++
.../apache/kylin/storage/cube/CubeScanner.java | 34 ++-
.../kylin/storage/cube/CubeTupleConverter.java | 14 +-
.../kylin/storage/hbase/CubeStorageQuery.java | 6 +
.../endpoint/protobuf/CubeVisit.proto | 38 ++++
23 files changed, 649 insertions(+), 485 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index 7a8c364..54d7525 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -1,25 +1,26 @@
package org.apache.kylin.cube.gridtable;
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Map;
-
-import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.gridtable.DefaultGTComparator;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.IGTCodeSystem;
import org.apache.kylin.gridtable.IGTComparator;
import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.apache.kylin.metadata.measure.serializer.DataTypeSerializer;
+import org.apache.kylin.metadata.measure.serializer.StringSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+
/**
* Created by shaoshi on 3/23/15.
* This implementation uses Dictionary to encode and decode the table; If a column doesn't have dictionary, will check
@@ -31,7 +32,8 @@ public class CubeCodeSystem implements IGTCodeSystem {
// ============================================================================
- private GTInfo info;
+ private transient GTInfo info;
+
private Map<Integer, Dictionary> dictionaryMap; // column index ==> dictionary of column
private Map<Integer, Integer> fixLenMap; // column index ==> fixed length of column
private Map<Integer, Integer> dependentMetricsMap;
@@ -39,9 +41,9 @@ public class CubeCodeSystem implements IGTCodeSystem {
private DataTypeSerializer[] serializers;
public CubeCodeSystem(Map<Integer, Dictionary> dictionaryMap) {
- this(dictionaryMap, Collections.<Integer, Integer>emptyMap(), Collections.<Integer, Integer>emptyMap());
+ this(dictionaryMap, Collections.<Integer, Integer> emptyMap(), Collections.<Integer, Integer> emptyMap());
}
-
+
public CubeCodeSystem(Map<Integer, Dictionary> dictionaryMap, Map<Integer, Integer> fixLenMap, Map<Integer, Integer> dependentMetricsMap) {
this.dictionaryMap = dictionaryMap;
this.fixLenMap = fixLenMap;
@@ -68,23 +70,7 @@ public class CubeCodeSystem implements IGTCodeSystem {
}
}
- this.comparator = new IGTComparator() {
- @Override
- public boolean isNull(ByteArray code) {
- // all 0xff is null
- byte[] array = code.array();
- for (int i = 0, j = code.offset(), n = code.length(); i < n; i++, j++) {
- if (array[j] != Dictionary.NULL)
- return false;
- }
- return true;
- }
-
- @Override
- public int compare(ByteArray code1, ByteArray code2) {
- return code1.compareTo(code2);
- }
- };
+ this.comparator = new DefaultGTComparator();
}
@Override
@@ -108,47 +94,33 @@ public class CubeCodeSystem implements IGTCodeSystem {
}
@Override
- public void encodeColumnValue(int col, Object value, int roundingFlag, ByteBuffer buf) {
- // this is a bit too complicated, but encoding only happens once at build time, so it is OK
+ public void encodeColumnValue(int col, Object value, int roundingFlag, ByteBuffer buf) {
DataTypeSerializer serializer = serializers[col];
- try {
- if (serializer instanceof DictionarySerializer) {
- ((DictionarySerializer) serializer).serializeWithRounding(value, roundingFlag, buf);
- } else {
- serializer.serialize(value, buf);
- }
- } catch (ClassCastException ex) {
- // try convert string into a correct object type
- try {
- if (value instanceof String) {
- Object converted = serializer.valueOf((String) value);
- if ((converted instanceof String) == false) {
- encodeColumnValue(col, converted, roundingFlag, buf);
- return;
- }
- }
- } catch (Throwable e) {
- logger.error("Fail to encode value '" + value + "'", e);
+ if (serializer instanceof DictionarySerializer) {
+ ((DictionarySerializer) serializer).serializeWithRounding(value, roundingFlag, buf);
+ } else {
+ if ((!(serializer instanceof StringSerializer || serializer instanceof FixLenSerializer)) && (value instanceof String)) {
+ value = serializer.valueOf((String) value);
}
- throw ex;
+ serializer.serialize(value, buf);
}
}
@Override
public Object decodeColumnValue(int col, ByteBuffer buf) {
- return serializers[col].deserialize(buf);
+ return serializers[col].deserialize(buf);
}
@Override
public MeasureAggregator<?>[] newMetricsAggregators(ImmutableBitSet columns, String[] aggrFunctions) {
assert columns.trueBitCount() == aggrFunctions.length;
-
+
MeasureAggregator<?>[] result = new MeasureAggregator[aggrFunctions.length];
for (int i = 0; i < result.length; i++) {
int col = columns.trueBitAt(i);
result[i] = MeasureAggregator.create(aggrFunctions[i], info.getColumnType(col).toString());
}
-
+
// deal with holistic distinct count
if (dependentMetricsMap != null) {
for (Integer child : dependentMetricsMap.keySet()) {
@@ -156,14 +128,14 @@ public class CubeCodeSystem implements IGTCodeSystem {
Integer parent = dependentMetricsMap.get(child);
if (columns.get(parent) == false)
throw new IllegalStateException();
-
+
int childIdx = columns.trueBitIndexOf(child);
int parentIdx = columns.trueBitIndexOf(parent);
result[childIdx].setDependentAggregator(result[parentIdx]);
}
}
}
-
+
return result;
}
@@ -217,7 +189,7 @@ public class CubeCodeSystem implements IGTCodeSystem {
FixLenSerializer(int fixLen) {
this.fixLen = fixLen;
}
-
+
private byte[] currentBuf() {
byte[] buf = current.get();
if (buf == null) {
@@ -254,11 +226,11 @@ public class CubeCodeSystem implements IGTCodeSystem {
while (tail > 0 && (buf[tail - 1] == RowConstants.ROWKEY_PLACE_HOLDER_BYTE || buf[tail - 1] == Dictionary.NULL)) {
tail--;
}
-
+
if (tail == 0) {
return buf[0] == Dictionary.NULL ? null : "";
}
-
+
return Bytes.toString(buf, 0, tail);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
index 9598a08..a0e0d05 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
@@ -17,31 +17,21 @@
package org.apache.kylin.cube.inmemcubing;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.Closeable;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.StandardOpenOption;
-import java.util.HashSet;
-import java.util.NoSuchElementException;
-
import org.apache.commons.io.IOUtils;
-import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.GTInfo;
-import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTRowBlock;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.StandardOpenOption;
+import java.util.HashSet;
+import java.util.NoSuchElementException;
+
/**
* A disk store that allows concurrent read and exclusive write.
*/
@@ -123,7 +113,7 @@ public class ConcurrentDiskStore implements IGTStore, Closeable {
}
@Override
- public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, ImmutableBitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException {
+ public IGTStoreScanner scan( GTScanRequest scanRequest) throws IOException {
return newReader();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
index d9b0ba6..99abd28 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
@@ -17,23 +17,6 @@
package org.apache.kylin.cube.inmemcubing;
-import static org.apache.kylin.common.util.MemoryBudgetController.*;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.Closeable;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.StandardOpenOption;
-import java.util.NoSuchElementException;
-
-import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.common.util.MemoryBudgetController.MemoryConsumer;
import org.apache.kylin.common.util.MemoryBudgetController.NotEnoughBudgetException;
@@ -45,6 +28,14 @@ import org.apache.kylin.gridtable.IGTStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.StandardOpenOption;
+import java.util.NoSuchElementException;
+
+import static org.apache.kylin.common.util.MemoryBudgetController.ONE_MB;
+
public class MemDiskStore implements IGTStore, Closeable {
private static final Logger logger = LoggerFactory.getLogger(MemDiskStore.class);
@@ -107,7 +98,7 @@ public class MemDiskStore implements IGTStore, Closeable {
}
@Override
- public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, ImmutableBitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException {
+ public IGTStoreScanner scan( GTScanRequest scanRequest) throws IOException {
synchronized (lock) {
return new Reader();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java b/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java
new file mode 100644
index 0000000..f7623e3
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java
@@ -0,0 +1,22 @@
+package org.apache.kylin.gridtable;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.dict.Dictionary;
+
+public class DefaultGTComparator implements IGTComparator {
+ @Override
+ public boolean isNull(ByteArray code) {
+ // all 0xff is null
+ byte[] array = code.array();
+ for (int i = 0, j = code.offset(), n = code.length(); i < n; i++, j++) {
+ if (array[j] != Dictionary.NULL)
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int compare(ByteArray code1, ByteArray code2) {
+ return code1.compareTo(code2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 38ccccd..9b53867 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -1,12 +1,6 @@
package org.apache.kylin.gridtable;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.Map.Entry;
-import java.util.SortedMap;
-
+import com.google.common.collect.Maps;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.measure.HLLCAggregator;
@@ -15,7 +9,12 @@ import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Maps;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.SortedMap;
@SuppressWarnings({ "rawtypes", "unchecked" })
public class GTAggregateScanner implements IGTScanner {
@@ -36,6 +35,7 @@ public class GTAggregateScanner implements IGTScanner {
throw new IllegalStateException();
this.info = inputScanner.getInfo();
+ //TODO bug? what if multiple metrics on same column? and what is this?
this.dimensions = req.getColumns().andNot(req.getAggrMetrics());
this.groupBy = req.getAggrGroupBy();
this.metrics = req.getAggrMetrics();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
index 87d7811..ff748ec 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
@@ -1,14 +1,14 @@
package org.apache.kylin.gridtable;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.metadata.model.TblColRef;
+
import java.util.Arrays;
import java.util.BitSet;
import java.util.Iterator;
import java.util.LinkedList;
-import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.model.DataType;
-import org.apache.kylin.metadata.model.TblColRef;
-
public class GTInfo {
public static Builder builder() {
@@ -16,6 +16,7 @@ public class GTInfo {
}
String tableName;
+
IGTCodeSystem codeSystem;
// column schema
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/core-cube/src/main/java/org/apache/kylin/gridtable/GTRawScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRawScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRawScanner.java
index 3e34bf4..6e59b05 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRawScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRawScanner.java
@@ -1,12 +1,12 @@
package org.apache.kylin.gridtable;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.gridtable.IGTStore.IGTStoreScanner;
+
import java.io.IOException;
import java.util.Iterator;
import java.util.NoSuchElementException;
-import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.gridtable.IGTStore.IGTStoreScanner;
-
public class GTRawScanner implements IGTScanner {
final GTInfo info;
@@ -22,8 +22,8 @@ public class GTRawScanner implements IGTScanner {
public GTRawScanner(GTInfo info, IGTStore store, GTScanRequest req) throws IOException {
this.info = info;
- this.selectedColBlocks = info.selectColumnBlocks(req.getColumns());
- this.storeScanner = store.scan(req.getPkStart(), req.getPkEnd(), selectedColBlocks, req);
+ this.selectedColBlocks = req.getSelectedColBlocks();
+ this.storeScanner = store.scan(req);
this.oneRecord = new GTRecord(info);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
index e6e00b7..83b0ed8 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
@@ -1,12 +1,11 @@
package org.apache.kylin.gridtable;
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.apache.kylin.metadata.measure.serializer.DataTypeSerializer;
+import java.nio.ByteBuffer;
+
@SuppressWarnings({ "rawtypes", "unchecked" })
/**
* This is just for example and is INCORRECT when numbers are encoded to bytes and compared in filter.
@@ -24,7 +23,7 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
public GTSampleCodeSystem() {
}
-
+
@Override
public void init(GTInfo info) {
this.info = info;
@@ -34,23 +33,7 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
this.serializers[i] = DataTypeSerializer.create(info.colTypes[i]);
}
- this.comparator = new IGTComparator() {
- @Override
- public boolean isNull(ByteArray code) {
- // all 0xff is null
- byte[] array = code.array();
- for (int i = 0, j = code.offset(), n = code.length(); i < n; i++, j++) {
- if (array[j] != (byte) 0xff)
- return false;
- }
- return true;
- }
-
- @Override
- public int compare(ByteArray code1, ByteArray code2) {
- return code1.compareTo(code2);
- }
- };
+ this.comparator = new DefaultGTComparator();
}
@Override
@@ -73,7 +56,7 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
@Override
public MeasureAggregator<?>[] newMetricsAggregators(ImmutableBitSet columns, String[] aggrFunctions) {
assert columns.trueBitCount() == aggrFunctions.length;
-
+
MeasureAggregator<?>[] result = new MeasureAggregator[aggrFunctions.length];
for (int i = 0; i < result.length; i++) {
int col = columns.trueBitAt(i);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index c5443b6..75cf8c4 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -1,13 +1,13 @@
package org.apache.kylin.gridtable;
-import java.util.Arrays;
-import java.util.Set;
-
+import com.google.common.collect.Sets;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
-import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Set;
public class GTScanRequest {
@@ -15,6 +15,7 @@ public class GTScanRequest {
private GTInfo info;
private GTScanRange range;
private ImmutableBitSet columns;
+ private ImmutableBitSet selectedColBlocks;
// optional filtering
private TupleFilter filterPushDown;
@@ -24,6 +25,10 @@ public class GTScanRequest {
private ImmutableBitSet aggrMetrics;
private String[] aggrMetricsFuncs;
+
+
+
+
public GTScanRequest(GTInfo info) {
this(info, null, null, null);
}
@@ -74,6 +79,8 @@ public class GTScanRequest {
if (columns == null)
columns = info.colAll;
+ this.selectedColBlocks = info.selectColumnBlocks(columns);
+
if (hasFilterPushDown()) {
validateFilterPushDown();
}
@@ -107,10 +114,22 @@ public class GTScanRequest {
}
}
+ public IGTScanner decorateScanner(IGTScanner scanner) throws IOException {
+ IGTScanner result = scanner;
+ if (this.hasFilterPushDown()) {
+ result = new GTFilterScanner(result, this);
+ }
+ if (this.hasAggregation()) {
+ result = new GTAggregateScanner(result, this);
+ }
+ return result;
+ }
+
public boolean hasFilterPushDown() {
return filterPushDown != null;
}
+ //TODO BUG? select sum() from fact, no aggr by
public boolean hasAggregation() {
return aggrGroupBy != null && aggrMetrics != null && aggrMetricsFuncs != null;
}
@@ -127,6 +146,12 @@ public class GTScanRequest {
return range.pkEnd;
}
+ public ImmutableBitSet getSelectedColBlocks()
+ {
+ return selectedColBlocks;
+ }
+
+
public ImmutableBitSet getColumns() {
return columns;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/core-cube/src/main/java/org/apache/kylin/gridtable/GridTable.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GridTable.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GridTable.java
index b8e59f6..8c0a375 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GridTable.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GridTable.java
@@ -34,14 +34,7 @@ public class GridTable implements Closeable {
public IGTScanner scan(GTScanRequest req) throws IOException {
IGTScanner result = new GTRawScanner(info, store, req);
-
- if (req.hasFilterPushDown()) {
- result = new GTFilterScanner(result, req);
- }
- if (req.hasAggregation()) {
- result = new GTAggregateScanner(result, req);
- }
- return result;
+ return req.decorateScanner(result);
}
public GTInfo getInfo() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/core-cube/src/main/java/org/apache/kylin/gridtable/IGTStore.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTStore.java b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTStore.java
index 70620b3..1d13795 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTStore.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTStore.java
@@ -4,23 +4,21 @@ import java.io.Closeable;
import java.io.IOException;
import java.util.Iterator;
-import org.apache.kylin.common.util.ImmutableBitSet;
-
public interface IGTStore {
GTInfo getInfo();
-
+
IGTStoreWriter rebuild(int shard) throws IOException;
-
+
IGTStoreWriter append(int shard, GTRowBlock.Writer fillLast) throws IOException;
-
- IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, ImmutableBitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException;
+
+ IGTStoreScanner scan(GTScanRequest scanRequest) throws IOException;
interface IGTStoreWriter extends Closeable {
void write(GTRowBlock block) throws IOException;
}
-
+
interface IGTStoreScanner extends Iterator<GTRowBlock>, Closeable {
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
index 0011d83..c7b9844 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
@@ -1,17 +1,15 @@
package org.apache.kylin.gridtable.memstore;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.GTInfo;
-import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTRowBlock;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTStore;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
public class GTSimpleMemStore implements IGTStore {
final GTInfo info;
@@ -78,7 +76,7 @@ public class GTSimpleMemStore implements IGTStore {
}
@Override
- public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, ImmutableBitSet selectedColBlocks, GTScanRequest additionalPushDown) {
+ public IGTStoreScanner scan( GTScanRequest scanRequest) {
return new IGTStoreScanner() {
Iterator<GTRowBlock> it = rowBlockList.iterator();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 881c9b2..9dfcc2e 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -18,12 +18,7 @@
package org.apache.kylin.dict;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
+import com.google.common.collect.Lists;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.metadata.MetadataManager;
@@ -37,7 +32,11 @@ import org.apache.kylin.source.TableSourceFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
public class DictionaryManager {
@@ -266,7 +265,7 @@ public class DictionaryManager {
if (dictInfos == null || dictInfos.isEmpty()) {
return null;
}
- //Collections.sort(dictInfos);
+ Collections.sort(dictInfos);
final List<DictionaryInfo> allResources = MetadataManager.getInstance(config).getStore().getAllResources(dictInfos.get(0), dictInfos.get(dictInfos.size() - 1), DictionaryInfo.class, DictionaryInfoSerializer.FULL_SERIALIZER);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
index c6b1a18..e72fe6c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
@@ -18,10 +18,6 @@
package org.apache.kylin.storage.tuple;
-import java.math.BigDecimal;
-import java.util.Date;
-import java.util.List;
-
import net.sf.ehcache.pool.sizeof.annotations.IgnoreSizeOf;
import org.apache.kylin.common.util.DateFormat;
@@ -30,6 +26,10 @@ import org.apache.kylin.metadata.measure.LongMutable;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.List;
+
/**
* @author xjiang
*/
@@ -190,6 +190,8 @@ public class Tuple implements ITuple {
return new BigDecimal(strValue);
} else if ("float".equals(dataTypeName)){
return Float.valueOf(strValue);
+ } else if ("boolean".equals(dataTypeName)) {
+ return Boolean.valueOf(strValue);
} else {
return strValue;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/server/src/main/resources/applicationContext.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/applicationContext.xml b/server/src/main/resources/applicationContext.xml
index b35b6fd..124c81d 100644
--- a/server/src/main/resources/applicationContext.xml
+++ b/server/src/main/resources/applicationContext.xml
@@ -1,112 +1,112 @@
-<beans xmlns="http://www.springframework.org/schema/beans"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns:context="http://www.springframework.org/schema/context"
- xmlns:mvc="http://www.springframework.org/schema/mvc"
- xmlns:task="http://www.springframework.org/schema/task"
- xmlns:aop="http://www.springframework.org/schema/aop"
- xmlns:cache="http://www.springframework.org/schema/cache"
- xmlns:p="http://www.springframework.org/schema/p"
- xsi:schemaLocation="http://www.springframework.org/schema/beans
- http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
- http://www.springframework.org/schema/context
- http://www.springframework.org/schema/context/spring-context-3.1.xsd
- http://www.springframework.org/schema/task
- http://www.springframework.org/schema/task/spring-task-3.1.xsd
- http://www.springframework.org/schema/mvc
- http://www.springframework.org/schema/mvc/spring-mvc-3.1.xsd
- http://www.springframework.org/schema/aop
- http://www.springframework.org/schema/aop/spring-aop-3.1.xsd
- http://www.springframework.org/schema/cache
- http://www.springframework.org/schema/cache/spring-cache.xsd">
-
- <description>Kylin Rest Service</description>
- <context:annotation-config/>
- <mvc:annotation-driven/>
- <aop:aspectj-autoproxy/>
-
- <context:component-scan base-package="org.apache.kylin.rest"/>
-
- <bean class="org.apache.kylin.rest.security.PasswordPlaceholderConfigurer">
- <property name="ignoreResourceNotFound" value="true"/>
- </bean>
-
-
- <!-- Rest service binding -->
- <bean
- class="org.springframework.web.servlet.mvc.annotation.DefaultAnnotationHandlerMapping"/>
-
- <bean id="mappingJacksonHttpMessageConverter"
- class="org.springframework.http.converter.json.MappingJackson2HttpMessageConverter"/>
- <bean id="stringHttpMessageConverter"
- class="org.springframework.http.converter.StringHttpMessageConverter"/>
- <bean id="formHttpMessageConverter"
- class="org.springframework.http.converter.FormHttpMessageConverter"/>
-
- <bean
- class="org.springframework.web.servlet.mvc.annotation.AnnotationMethodHandlerAdapter">
- <property name="messageConverters">
- <list>
- <ref bean="mappingJacksonHttpMessageConverter"/>
- <ref bean="stringHttpMessageConverter"/>
- <ref bean="formHttpMessageConverter"/>
- </list>
- </property>
- </bean>
-
- <bean
- class="org.springframework.web.servlet.view.ContentNegotiatingViewResolver">
- <property name="mediaTypes">
- <map>
- <entry key="html" value="text/html"/>
- <entry key="json" value="application/json"/>
- </map>
- </property>
- <property name="viewResolvers">
- <list>
- <bean class="org.springframework.web.servlet.view.BeanNameViewResolver"/>
- <bean
- class="org.springframework.web.servlet.view.InternalResourceViewResolver">
- <!-- <property name="prefix" value="/WEB-INF/jsp/"/> -->
- <property name="suffix" value=".jsp"/>
- </bean>
- </list>
- </property>
- <property name="defaultViews">
- <list>
- <bean
- class="org.springframework.web.servlet.view.json.MappingJackson2JsonView"/>
- </list>
- </property>
- </bean>
- <!-- Rest service binding -->
- <task:annotation-driven/>
- <bean id="performanceMonitorTask" class="org.apache.kylin.rest.util.PerformanceMonitorTask"></bean>
-
- <!-- Cache Config -->
- <cache:annotation-driven/>
-
- <bean id="cacheManager" class="org.springframework.cache.ehcache.EhCacheCacheManager"
- p:cacheManager-ref="ehcache"/>
- <beans profile="default">
- <bean id="ehcache"
- class="org.springframework.cache.ehcache.EhCacheManagerFactoryBean"
- p:configLocation="classpath:ehcache.xml" p:shared="true"/>
- <bean id="cacheUpdater" class="org.apache.kylin.common.cache.RemoteCacheUpdater"/>
- </beans>
- <beans profile="sandbox,testing">
- <bean id="ehcache"
- class="org.springframework.cache.ehcache.EhCacheManagerFactoryBean"
- p:configLocation="classpath:ehcache-test.xml" p:shared="true"/>
- <bean id="cacheUpdater" class="org.apache.kylin.common.cache.LocalCacheUpdater"/>
- </beans>
-
- <!-- hbase storage/global lock Config -->
- <beans profile="default,sandbox">
- <bean id="aclHBaseStorage" class="org.apache.kylin.rest.security.RealAclHBaseStorage"/>
- <bean id="jobLock" class="org.apache.kylin.storage.hbase.util.ZookeeperJobLock"/>
- </beans>
- <beans profile="testing">
- <bean id="aclHBaseStorage" class="org.apache.kylin.rest.security.MockAclHBaseStorage"/>
- <bean id="jobLock" class="org.apache.kylin.job.lock.MockJobLock"/>
- </beans>
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:mvc="http://www.springframework.org/schema/mvc"
+ xmlns:task="http://www.springframework.org/schema/task"
+ xmlns:aop="http://www.springframework.org/schema/aop"
+ xmlns:cache="http://www.springframework.org/schema/cache"
+ xmlns:p="http://www.springframework.org/schema/p"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context-3.1.xsd
+ http://www.springframework.org/schema/task
+ http://www.springframework.org/schema/task/spring-task-3.1.xsd
+ http://www.springframework.org/schema/mvc
+ http://www.springframework.org/schema/mvc/spring-mvc-3.1.xsd
+ http://www.springframework.org/schema/aop
+ http://www.springframework.org/schema/aop/spring-aop-3.1.xsd
+ http://www.springframework.org/schema/cache
+ http://www.springframework.org/schema/cache/spring-cache.xsd">
+
+ <description>Kylin Rest Service</description>
+ <context:annotation-config/>
+ <mvc:annotation-driven/>
+ <aop:aspectj-autoproxy/>
+
+ <context:component-scan base-package="org.apache.kylin.rest"/>
+
+ <bean class="org.apache.kylin.rest.security.PasswordPlaceholderConfigurer">
+ <property name="ignoreResourceNotFound" value="true"/>
+ </bean>
+
+
+ <!-- Rest service binding -->
+ <bean
+ class="org.springframework.web.servlet.mvc.annotation.DefaultAnnotationHandlerMapping"/>
+
+ <bean id="mappingJacksonHttpMessageConverter"
+ class="org.springframework.http.converter.json.MappingJackson2HttpMessageConverter"/>
+ <bean id="stringHttpMessageConverter"
+ class="org.springframework.http.converter.StringHttpMessageConverter"/>
+ <bean id="formHttpMessageConverter"
+ class="org.springframework.http.converter.FormHttpMessageConverter"/>
+
+ <bean
+ class="org.springframework.web.servlet.mvc.annotation.AnnotationMethodHandlerAdapter">
+ <property name="messageConverters">
+ <list>
+ <ref bean="mappingJacksonHttpMessageConverter"/>
+ <ref bean="stringHttpMessageConverter"/>
+ <ref bean="formHttpMessageConverter"/>
+ </list>
+ </property>
+ </bean>
+
+ <bean
+ class="org.springframework.web.servlet.view.ContentNegotiatingViewResolver">
+ <property name="mediaTypes">
+ <map>
+ <entry key="html" value="text/html"/>
+ <entry key="json" value="application/json"/>
+ </map>
+ </property>
+ <property name="viewResolvers">
+ <list>
+ <bean class="org.springframework.web.servlet.view.BeanNameViewResolver"/>
+ <bean
+ class="org.springframework.web.servlet.view.InternalResourceViewResolver">
+ <!-- <property name="prefix" value="/WEB-INF/jsp/"/> -->
+ <property name="suffix" value=".jsp"/>
+ </bean>
+ </list>
+ </property>
+ <property name="defaultViews">
+ <list>
+ <bean
+ class="org.springframework.web.servlet.view.json.MappingJackson2JsonView"/>
+ </list>
+ </property>
+ </bean>
+ <!-- Rest service binding -->
+ <task:annotation-driven/>
+ <bean id="performanceMonitorTask" class="org.apache.kylin.rest.util.PerformanceMonitorTask"></bean>
+
+ <!-- Cache Config -->
+ <cache:annotation-driven/>
+
+ <bean id="cacheManager" class="org.springframework.cache.ehcache.EhCacheCacheManager"
+ p:cacheManager-ref="ehcache"/>
+ <beans profile="default">
+ <bean id="ehcache"
+ class="org.springframework.cache.ehcache.EhCacheManagerFactoryBean"
+ p:configLocation="classpath:ehcache.xml" p:shared="true"/>
+ <bean id="cacheUpdater" class="org.apache.kylin.common.cache.RemoteCacheUpdater"/>
+ </beans>
+ <beans profile="sandbox,testing">
+ <bean id="ehcache"
+ class="org.springframework.cache.ehcache.EhCacheManagerFactoryBean"
+ p:configLocation="classpath:ehcache-test.xml" p:shared="true"/>
+ <bean id="cacheUpdater" class="org.apache.kylin.common.cache.LocalCacheUpdater"/>
+ </beans>
+
+ <!-- hbase storage/global lock Config -->
+ <beans profile="default,sandbox">
+ <bean id="aclHBaseStorage" class="org.apache.kylin.rest.security.RealAclHBaseStorage"/>
+ <bean id="jobLock" class="org.apache.kylin.storage.hbase.util.ZookeeperJobLock"/>
+ </beans>
+ <beans profile="testing">
+ <bean id="aclHBaseStorage" class="org.apache.kylin.rest.security.MockAclHBaseStorage"/>
+ <bean id="jobLock" class="org.apache.kylin.job.lock.MockJobLock"/>
+ </beans>
</beans>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseEndpointRPC.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseEndpointRPC.java
new file mode 100644
index 0000000..8c7b24c
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseEndpointRPC.java
@@ -0,0 +1,126 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ * /
+ */
+
+package org.apache.kylin.storage.cube;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.gridtable.*;
+import org.apache.kylin.storage.hbase.HBaseConnection;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * for test use only
+ */
+public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
+
+ public CubeHBaseEndpointRPC(GTInfo info, CubeSegment cubeSeg, Cuboid cuboid) {
+ super(info, cubeSeg, cuboid);
+ }
+
+ @Override
+ public IGTScanner getGTScanner(GTScanRequest scanRequest) throws IOException {
+
+ // primary key (also the 0th column block) is always selected
+ final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
+
+ // globally shared connection, does not require close
+ HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+
+ final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
+ final List<Pair<byte[], byte[]>> hbaseColumns = makeHBaseColumns(selectedColBlocks);
+
+ Scan hbaseScan = buildScan(scanRequest.getPkStart(), scanRequest.getPkEnd(), hbaseColumns);
+ final ResultScanner scanner = hbaseTable.getScanner(hbaseScan);
+ final Iterator<Result> iterator = scanner.iterator();
+
+ final GTRowBlock oneBlock = new GTRowBlock(scanRequest.getInfo()); // avoid object creation
+ oneBlock.setNumberOfRows(1); // row block is always disabled for cubes, row block contains only one record
+
+ IGTStore store = new IGTStore() {
+ @Override
+ public GTInfo getInfo() {
+ return info;
+ }
+
+ @Override
+ public IGTStoreWriter rebuild(int shard) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IGTStoreWriter append(int shard, GTRowBlock.Writer fillLast) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IGTStoreScanner scan(GTScanRequest scanRequest) throws IOException {
+ return new IGTStoreScanner() {
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public GTRowBlock next() {
+ Result result = iterator.next();
+
+ // dimensions, set to primary key, also the 0th column block
+ byte[] rowkey = result.getRow();
+ oneBlock.getPrimaryKey().set(rowkey, RowConstants.ROWKEY_CUBOIDID_LEN, rowkey.length - RowConstants.ROWKEY_CUBOIDID_LEN);
+ oneBlock.getCellBlock(0).set(rowkey, RowConstants.ROWKEY_CUBOIDID_LEN, rowkey.length - RowConstants.ROWKEY_CUBOIDID_LEN);
+
+ // metrics
+ int hbaseColIdx = 0;
+ for (int i = 1; i < selectedColBlocks.trueBitCount(); i++) {
+ int colBlockIdx = selectedColBlocks.trueBitAt(i);
+ Pair<byte[], byte[]> hbaseColumn = hbaseColumns.get(hbaseColIdx++);
+ Cell cell = result.getColumnLatestCell(hbaseColumn.getFirst(), hbaseColumn.getSecond());
+ oneBlock.getCellBlock(colBlockIdx).set(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ }
+ return oneBlock;
+
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close() throws IOException {
+ scanner.close();
+ }
+ };
+ }
+ };
+
+ GTRawScanner rawScanner = new GTRawScanner(scanRequest.getInfo(), store, scanRequest);
+ return scanRequest.decorateScanner(rawScanner);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseRPC.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseRPC.java
new file mode 100644
index 0000000..0045355
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseRPC.java
@@ -0,0 +1,95 @@
+package org.apache.kylin.storage.cube;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.cube.model.HBaseColumnDesc;
+import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
+import org.apache.kylin.cube.model.HBaseMappingDesc;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.gridtable.IGTScanner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+public abstract class CubeHBaseRPC {
+
+ public static final Logger logger = LoggerFactory.getLogger(CubeHBaseRPC.class);
+
+ public static final int SCAN_CACHE = 1024;
+
+ final protected GTInfo info;
+ final protected CubeSegment cubeSeg;
+ final protected Cuboid cuboid;
+
+ public CubeHBaseRPC(GTInfo info, CubeSegment cubeSeg, Cuboid cuboid) {
+ this.info = info;
+ this.cubeSeg = cubeSeg;
+ this.cuboid = cuboid;
+ }
+
+ abstract IGTScanner getGTScanner(GTScanRequest scanRequest) throws IOException;
+
+ protected Scan buildScan(GTRecord pkStart, GTRecord pkEnd, List<Pair<byte[], byte[]>> selectedColumns) {
+ Scan scan = new Scan();
+ scan.setCaching(SCAN_CACHE);
+ scan.setCacheBlocks(true);
+ scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
+
+ for (Pair<byte[], byte[]> hbaseColumn : selectedColumns) {
+ byte[] byteFamily = hbaseColumn.getFirst();
+ byte[] byteQualifier = hbaseColumn.getSecond();
+ scan.addColumn(byteFamily, byteQualifier);
+ }
+
+ scan.setStartRow(makeRowKeyToScan(pkStart, (byte) 0x00));
+ scan.setStopRow(makeRowKeyToScan(pkEnd, (byte) 0xff));
+ //TODO fuzzy match
+ return scan;
+ }
+
+ protected byte[] makeRowKeyToScan(GTRecord pkRec, byte fill) {
+ ByteArray pk = GTRecord.exportScanKey(pkRec);
+ int pkMaxLen = info.getMaxColumnLength(info.getPrimaryKey());
+
+ byte[] buf = new byte[pkMaxLen + RowConstants.ROWKEY_CUBOIDID_LEN];
+ Arrays.fill(buf, fill);
+
+ System.arraycopy(cuboid.getBytes(), 0, buf, 0, RowConstants.ROWKEY_CUBOIDID_LEN);
+ if (pk != null && pk.array() != null) {
+ System.arraycopy(pk.array(), pk.offset(), buf, RowConstants.ROWKEY_CUBOIDID_LEN, pk.length());
+ }
+ return buf;
+ }
+
+ protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) {
+ List<Pair<byte[], byte[]>> result = Lists.newArrayList();
+
+ int colBlockIdx = 1; // start from 1; the 0th column block is primary key which maps to rowkey
+ HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
+ for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
+ byte[] byteFamily = Bytes.toBytes(familyDesc.getName());
+ for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
+ if (selectedColBlocks.get(colBlockIdx)) {
+ byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier());
+ result.add(new Pair<byte[], byte[]>(byteFamily, byteQualifier));
+ }
+ colBlockIdx++;
+ }
+ }
+
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
deleted file mode 100644
index 8001fbd..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
+++ /dev/null
@@ -1,177 +0,0 @@
-package org.apache.kylin.storage.cube;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.model.HBaseColumnDesc;
-import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
-import org.apache.kylin.cube.model.HBaseMappingDesc;
-import org.apache.kylin.gridtable.GTInfo;
-import org.apache.kylin.gridtable.GTRecord;
-import org.apache.kylin.gridtable.GTRowBlock;
-import org.apache.kylin.gridtable.GTScanRequest;
-import org.apache.kylin.gridtable.IGTStore;
-import org.apache.kylin.gridtable.GTRowBlock.Writer;
-import org.apache.kylin.storage.hbase.HBaseConnection;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-public class CubeHBaseReadonlyStore implements IGTStore {
-
- public static final Logger logger = LoggerFactory.getLogger(CubeHBaseReadonlyStore.class);
-
- public static final int SCAN_CACHE = 1024;
-
- final private GTInfo info;
- final private CubeSegment cubeSeg;
- final private Cuboid cuboid;
-
- public CubeHBaseReadonlyStore(GTInfo info, CubeSegment cubeSeg, Cuboid cuboid) {
- this.info = info;
- this.cubeSeg = cubeSeg;
- this.cuboid = cuboid;
- }
-
- @Override
- public GTInfo getInfo() {
- return info;
- }
-
- @Override
- public IGTStoreWriter rebuild(int shard) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public IGTStoreWriter append(int shard, Writer fillLast) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, ImmutableBitSet selectedColumnBlocks, GTScanRequest additionalPushDown) throws IOException {
- // TODO enable coprocessor
-
- // primary key (also the 0th column block) is always selected
- final ImmutableBitSet selectedColBlocks = selectedColumnBlocks.set(0);
-
- // globally shared connection, does not require close
- HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
-
- final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
- final List<Pair<byte[], byte[]>> hbaseColumns = makeHBaseColumns(selectedColBlocks);
-
- Scan hbaseScan = buildScan(pkStart, pkEnd, hbaseColumns);
- final ResultScanner scanner = hbaseTable.getScanner(hbaseScan);
- final Iterator<Result> iterator = scanner.iterator();
- final GTRowBlock oneBlock = new GTRowBlock(info); // avoid object creation
- oneBlock.setNumberOfRows(1); // row block is always disabled for cubes, row block contains only one record
-
- return new IGTStoreScanner() {
-
- @Override
- public boolean hasNext() {
- return iterator.hasNext();
- }
-
- @Override
- public GTRowBlock next() {
- Result result = iterator.next();
-
- // dimensions, set to primary key, also the 0th column block
- byte[] rowkey = result.getRow();
- oneBlock.getPrimaryKey().set(rowkey, RowConstants.ROWKEY_CUBOIDID_LEN, rowkey.length - RowConstants.ROWKEY_CUBOIDID_LEN);
- oneBlock.getCellBlock(0).set(rowkey, RowConstants.ROWKEY_CUBOIDID_LEN, rowkey.length - RowConstants.ROWKEY_CUBOIDID_LEN);
-
- // metrics
- int hbaseColIdx = 0;
- for (int i = 0; i < selectedColBlocks.trueBitCount(); i++) {
- int colBlockIdx = selectedColBlocks.trueBitAt(i);
- Pair<byte[], byte[]> hbaseColumn = hbaseColumns.get(hbaseColIdx++);
- Cell cell = result.getColumnLatestCell(hbaseColumn.getFirst(), hbaseColumn.getSecond());
- oneBlock.getCellBlock(colBlockIdx).set(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
- }
- return oneBlock;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close() throws IOException {
- scanner.close();
- hbaseTable.close();
- }
- };
- }
-
- private Scan buildScan(GTRecord pkStart, GTRecord pkEnd, List<Pair<byte[], byte[]>> selectedColumns) {
- Scan scan = new Scan();
- scan.setCaching(SCAN_CACHE);
- scan.setCacheBlocks(true);
- scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
-
- for (Pair<byte[], byte[]> hbaseColumn : selectedColumns) {
- byte[] byteFamily = hbaseColumn.getFirst();
- byte[] byteQualifier = hbaseColumn.getSecond();
- scan.addColumn(byteFamily, byteQualifier);
- }
-
- scan.setStartRow(makeRowKeyToScan(pkStart, (byte) 0x00));
- scan.setStopRow(makeRowKeyToScan(pkEnd, (byte) 0xff));
- //TODO fuzzy match
- return scan;
- }
-
- private byte[] makeRowKeyToScan(GTRecord pkRec, byte fill) {
- ByteArray pk = GTRecord.exportScanKey(pkRec);
- int pkMaxLen = info.getMaxColumnLength(info.getPrimaryKey());
-
- byte[] buf = new byte[pkMaxLen + RowConstants.ROWKEY_CUBOIDID_LEN];
- Arrays.fill(buf, fill);
-
- System.arraycopy(cuboid.getBytes(), 0, buf, 0, RowConstants.ROWKEY_CUBOIDID_LEN);
- if (pk != null && pk.array() != null) {
- System.arraycopy(pk.array(), pk.offset(), buf, RowConstants.ROWKEY_CUBOIDID_LEN, pk.length());
- }
- return buf;
- }
-
- private List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) {
- List<Pair<byte[], byte[]>> result = Lists.newArrayList();
-
- int colBlockIdx = 1; // start from 1; the 0th column block is primary key which maps to rowkey
- HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
- for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
- byte[] byteFamily = Bytes.toBytes(familyDesc.getName());
- for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
- if (selectedColBlocks.get(colBlockIdx)) {
- byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier());
- result.add(new Pair<byte[], byte[]>(byteFamily, byteQualifier));
- }
- colBlockIdx++;
- }
- }
-
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseScanRPC.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseScanRPC.java
new file mode 100644
index 0000000..0251c0f
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseScanRPC.java
@@ -0,0 +1,106 @@
+package org.apache.kylin.storage.cube;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.gridtable.*;
+import org.apache.kylin.storage.hbase.HBaseConnection;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * for test use only
+ */
+public class CubeHBaseScanRPC extends CubeHBaseRPC {
+
+ public CubeHBaseScanRPC(GTInfo info, CubeSegment cubeSeg, Cuboid cuboid) {
+ super(info, cubeSeg, cuboid);
+ }
+
+ @Override
+ public IGTScanner getGTScanner(GTScanRequest scanRequest) throws IOException {
+
+ // primary key (also the 0th column block) is always selected
+ final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
+
+ // globally shared connection, does not require close
+ HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+
+ final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
+ final List<Pair<byte[], byte[]>> hbaseColumns = makeHBaseColumns(selectedColBlocks);
+
+ Scan hbaseScan = buildScan(scanRequest.getPkStart(), scanRequest.getPkEnd(), hbaseColumns);
+ final ResultScanner scanner = hbaseTable.getScanner(hbaseScan);
+ final Iterator<Result> iterator = scanner.iterator();
+
+ final GTRowBlock oneBlock = new GTRowBlock(scanRequest.getInfo()); // avoid object creation
+ oneBlock.setNumberOfRows(1); // row block is always disabled for cubes, row block contains only one record
+
+ IGTStore store = new IGTStore() {
+ @Override
+ public GTInfo getInfo() {
+ return info;
+ }
+
+ @Override
+ public IGTStoreWriter rebuild(int shard) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IGTStoreWriter append(int shard, GTRowBlock.Writer fillLast) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IGTStoreScanner scan(GTScanRequest scanRequest) throws IOException {
+ return new IGTStoreScanner() {
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public GTRowBlock next() {
+ Result result = iterator.next();
+
+ // dimensions, set to primary key, also the 0th column block
+ byte[] rowkey = result.getRow();
+ oneBlock.getPrimaryKey().set(rowkey, RowConstants.ROWKEY_CUBOIDID_LEN, rowkey.length - RowConstants.ROWKEY_CUBOIDID_LEN);
+ oneBlock.getCellBlock(0).set(rowkey, RowConstants.ROWKEY_CUBOIDID_LEN, rowkey.length - RowConstants.ROWKEY_CUBOIDID_LEN);
+
+ // metrics
+ int hbaseColIdx = 0;
+ for (int i = 1; i < selectedColBlocks.trueBitCount(); i++) {
+ int colBlockIdx = selectedColBlocks.trueBitAt(i);
+ Pair<byte[], byte[]> hbaseColumn = hbaseColumns.get(hbaseColIdx++);
+ Cell cell = result.getColumnLatestCell(hbaseColumn.getFirst(), hbaseColumn.getSecond());
+ oneBlock.getCellBlock(colBlockIdx).set(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ }
+ return oneBlock;
+
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close() throws IOException {
+ scanner.close();
+ }
+ };
+ }
+ };
+
+ GTRawScanner rawScanner = new GTRawScanner(scanRequest.getInfo(), store, scanRequest);
+ return scanRequest.decorateScanner(rawScanner);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/storage/src/main/java/org/apache/kylin/storage/cube/CubeScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeScanner.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeScanner.java
index 6af83d1..917a1a8 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeScanner.java
@@ -1,20 +1,12 @@
package org.apache.kylin.storage.cube;
-import java.io.IOException;
-import java.util.BitSet;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.Set;
-
+import com.google.common.collect.Lists;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.gridtable.CubeGridTable;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
import org.apache.kylin.gridtable.GTInfo;
-import org.apache.kylin.gridtable.GTRawScanner;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRange;
import org.apache.kylin.gridtable.GTScanRangePlanner;
@@ -25,7 +17,8 @@ import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.*;
public class CubeScanner implements IGTScanner {
@@ -33,15 +26,15 @@ public class CubeScanner implements IGTScanner {
final CubeSegment cubeSeg;
final GTInfo info;
- final CubeHBaseReadonlyStore store;
final List<GTScanRequest> scanRequests;
final Scanner scanner;
+ final Cuboid cuboid;
public CubeScanner(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, //
Collection<FunctionDesc> metrics, TupleFilter filter) {
+ this.cuboid = cuboid;
this.cubeSeg = cubeSeg;
this.info = CubeGridTable.newGTInfo(cubeSeg, cuboid.getId());
- this.store = new CubeHBaseReadonlyStore(info, cubeSeg, cuboid);
CuboidToGridTableMapping mapping = new CuboidToGridTableMapping(cuboid);
TupleFilter gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, mapping.getCuboidDimensionsInGTOrder(), groups);
@@ -57,7 +50,7 @@ public class CubeScanner implements IGTScanner {
for (GTScanRange range : scanRanges) {
scanRequests.add(new GTScanRequest(info, range, gtDimensions, gtAggrGroups, gtAggrMetrics, gtAggrFuncs, gtFilter));
}
-
+
scanner = new Scanner();
}
@@ -129,25 +122,26 @@ public class CubeScanner implements IGTScanner {
public boolean hasNext() {
if (next != null)
return true;
-
+
if (curIterator == null) {
if (cur >= scanRequests.size())
return false;
try {
- inputScanners[cur] = new GTRawScanner(info, store, scanRequests.get(cur));
+ CubeHBaseScanRPC rpc = new CubeHBaseScanRPC(info, cubeSeg, cuboid);
+ inputScanners[cur] = rpc.getGTScanner(scanRequests.get(cur));
curIterator = inputScanners[cur].iterator();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
-
+
if (curIterator.hasNext() == false) {
curIterator = null;
cur++;
return hasNext();
}
-
+
next = curIterator.next();
return true;
}
@@ -186,7 +180,7 @@ public class CubeScanner implements IGTScanner {
for (int i = 0; i < inputScanners.length; i++) {
if (inputScanners[i] == null)
break;
-
+
result += inputScanners[i].getScannedRowCount();
}
return result;
@@ -197,12 +191,12 @@ public class CubeScanner implements IGTScanner {
for (int i = 0; i < inputScanners.length; i++) {
if (inputScanners[i] == null)
break;
-
+
result += inputScanners[i].getScannedRowBlockCount();
}
return result;
}
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/storage/src/main/java/org/apache/kylin/storage/cube/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeTupleConverter.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeTupleConverter.java
index 0657a69..7bcd0df 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeTupleConverter.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeTupleConverter.java
@@ -18,11 +18,7 @@
package org.apache.kylin.storage.cube;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
+import com.google.common.collect.Lists;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -36,8 +32,14 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.tuple.Tuple;
import org.apache.kylin.storage.tuple.TupleInfo;
-import com.google.common.collect.Lists;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+/**
+ * convert GTRecord to tuple
+ */
public class CubeTupleConverter {
final CubeSegment cubeSeg;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageQuery.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageQuery.java
index 4c3920b..459e72d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageQuery.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageQuery.java
@@ -405,6 +405,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
logger.info("READY segs count: " + segs.size());
// build row key range for each cube segment
+ StringBuilder sb = new StringBuilder("hbasekeyrange trace: ");
for (CubeSegment cubeSeg : segs) {
// consider derived (lookup snapshot), filter on dimension may
@@ -420,11 +421,16 @@ public class CubeStorageQuery implements ICachableStorageQuery {
scanRanges.add(rowKeyRange);
}
+ sb.append(scanRanges.size() + "=>");
List<HBaseKeyRange> mergedRanges = mergeOverlapRanges(scanRanges);
+ sb.append(mergedRanges.size() + "=>");
mergedRanges = mergeTooManyRanges(mergedRanges);
+ sb.append(mergedRanges.size() + ", ");
result.addAll(mergedRanges);
}
+ logger.info(sb.toString());
+
logger.info("hbasekeyrange count: " + result.size());
dropUnhitSegments(result);
logger.info("hbasekeyrange count after dropping unhit :" + result.size());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dad5e8ac/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/protobuf/CubeVisit.proto
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/protobuf/CubeVisit.proto b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/protobuf/CubeVisit.proto
new file mode 100644
index 0000000..a7fb4b7
--- /dev/null
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/protobuf/CubeVisit.proto
@@ -0,0 +1,38 @@
+
+// usage:
+// protoc --java_out=./storage/src/main/java ./storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/protobuf/CubeVisit.proto
+
+option java_package = "org.apache.kylin.storage.hbase.coprocessor.endpoint.generated";
+option java_outer_classname = "CubeVisitProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+message CubeVisitRequest {
+ required bytes scanRequest = 1;
+}
+
+message CubeVisitResponseUncompressed {
+ message Record {
+ required bytes content = 1;
+ }
+ //all entries in this struct be optional to conveniently add more entries in the future
+ message Stats {
+ optional int64 serviceStartTime = 1;
+ optional int64 serviceEndTime = 2;
+ optional int32 scannedRowCount = 3;
+ optional int32 aggregatedRowCount = 4;
+ }
+ repeated Record records = 1;
+ required Stats stats = 2;
+}
+
+message CubeVisitResponse
+{
+ required bytes compressed = 1;
+}
+
+service CubeVisitService {
+ rpc visitCube(CubeVisitRequest)
+ returns (CubeVisitResponse);
+}