You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/09/29 04:43:00 UTC
[1/2] incubator-kylin git commit: try storage v2 on shard
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-942 7c4847678 -> 62b536b0d
try storage v2 on shard
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7b9cbcbb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7b9cbcbb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7b9cbcbb
Branch: refs/heads/KYLIN-942
Commit: 7b9cbcbb0e9d7c7ef8d97d0ae4da8941f8b5ff82
Parents: 7c48476
Author: honma <ho...@ebay.com>
Authored: Mon Sep 28 17:55:52 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Sep 28 17:55:52 2015 +0800
----------------------------------------------------------------------
.../kylin/storage/hbase/HBaseStorage.java | 2 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 84 +++++++++++++-------
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 26 +++++-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 30 +++++--
.../storage/hbase/cube/v2/CubeScanner.java | 2 +-
.../hbase/cube/v2/HBaseReadonlyStore.java | 2 +-
6 files changed, 103 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7b9cbcbb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
index bbcbe7f..53465d8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
@@ -43,7 +43,7 @@ import com.google.common.base.Preconditions;
public class HBaseStorage implements IStorage {
private final static boolean allowStorageLayerCache = true;
- private final static String defaultCubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v1.CubeStorageQuery";
+ private final static String defaultCubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v2.CubeStorageQuery";
private final static String defaultIIStorageQuery = "org.apache.kylin.storage.hbase.ii.InvertedIndexStorageQuery";
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7b9cbcbb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 7d1d833..93a12ce 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -21,9 +21,13 @@ package org.apache.kylin.storage.hbase.cube.v2;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collection;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
import java.util.zip.DataFormatException;
import javax.annotation.Nullable;
@@ -43,13 +47,14 @@ import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.steps.HBaseConnection;
import com.google.common.base.Function;
import com.google.common.collect.Collections2;
import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
import com.google.protobuf.ByteString;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
@@ -122,40 +127,61 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
@Override
public IGTScanner getGTScanner(final GTScanRequest scanRequest) throws IOException {
- try {
- // primary key (also the 0th column block) is always selected
- final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
- // globally shared connection, does not require close
- HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
- final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
- final List<Pair<byte[], byte[]>> hbaseColumns = makeHBaseColumns(selectedColBlocks);
-
- RawScan rawScan = prepareRawScan(scanRequest.getPkStart(), scanRequest.getPkEnd(), hbaseColumns);
-
- byte[] scanRequestBytes = KryoUtils.serialize(scanRequest);
- byte[] rawScanBytes = KryoUtils.serialize(rawScan);
- CubeVisitProtos.CubeVisitRequest.Builder builder = CubeVisitProtos.CubeVisitRequest.newBuilder();
- builder.setGtScanRequest(ByteString.copyFrom(scanRequestBytes)).setHbaseRawScan(ByteString.copyFrom(rawScanBytes));
-
- Collection<CubeVisitProtos.CubeVisitResponse> results = getResults(builder.build(), hbaseTable, rawScan.startKey, rawScan.endKey);
- final Collection<byte[]> rowBlocks = Collections2.transform(results, new Function<CubeVisitProtos.CubeVisitResponse, byte[]>() {
- @Nullable
+ // primary key (also the 0th column block) is always selected
+ final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
+ // globally shared connection, does not require close
+ HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
+ final List<Pair<byte[], byte[]>> hbaseColumns = makeHBaseColumns(selectedColBlocks);
+
+ List<RawScan> rawScans = prepareRawScan(scanRequest.getPkStart(), scanRequest.getPkEnd(), hbaseColumns);
+
+ byte[] scanRequestBytes = KryoUtils.serialize(scanRequest);
+ final ByteString scanRequestBytesString = ByteString.copyFrom(scanRequestBytes);
+
+ ExecutorService executorService = Executors.newFixedThreadPool(rawScans.size());
+ final List<byte[]> rowBlocks = Collections.synchronizedList(Lists.<byte[]> newArrayList());
+
+ for (final RawScan rawScan : rawScans) {
+ executorService.submit(new Runnable() {
@Override
- public byte[] apply(CubeVisitProtos.CubeVisitResponse input) {
+ public void run() {
+ final byte[] rawScanBytes = KryoUtils.serialize(rawScan);
+ CubeVisitProtos.CubeVisitRequest.Builder builder = CubeVisitProtos.CubeVisitRequest.newBuilder();
+ builder.setGtScanRequest(scanRequestBytesString).setHbaseRawScan(ByteString.copyFrom(rawScanBytes));
+
+ Collection<CubeVisitProtos.CubeVisitResponse> results;
try {
- return CompressionUtils.decompress(input.getCompressedRows().toByteArray());
- } catch (IOException | DataFormatException e) {
- throw new RuntimeException(e);
+ results = getResults(builder.build(), hbaseTable, rawScan.startKey, rawScan.endKey);
+ } catch (Throwable throwable) {
+ throw new RuntimeException("Error when visiting cubes by endpoint:", throwable);
}
+
+ Collection<byte[]> part = Collections2.transform(results, new Function<CubeVisitProtos.CubeVisitResponse, byte[]>() {
+ @Nullable
+ @Override
+ public byte[] apply(CubeVisitProtos.CubeVisitResponse input) {
+ try {
+ return CompressionUtils.decompress(input.getCompressedRows().toByteArray());
+ } catch (IOException | DataFormatException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ rowBlocks.addAll(part);
}
});
-
- return new EndpintResultsAsGTScanner(fullGTInfo, rowBlocks.iterator());
-
- } catch (Throwable throwable) {
- throwable.printStackTrace();
}
- return null;
+ executorService.shutdown();
+ try {
+ if (!executorService.awaitTermination(1, TimeUnit.HOURS)) {
+ throw new RuntimeException("Visiting cube by endpoint timeout");
+ }
+ } catch (InterruptedException e) {
+ throw new RuntimeException("Visiting cube by endpoint gets interrupted");
+ }
+
+ return new EndpintResultsAsGTScanner(fullGTInfo, rowBlocks.iterator());
}
//TODO : async callback
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7b9cbcbb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 09bef0f..83d1fea 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -14,6 +14,7 @@ import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.common.util.ShardingHash;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowConstants;
@@ -69,13 +70,27 @@ public abstract class CubeHBaseRPC {
return scan;
}
- protected RawScan prepareRawScan(GTRecord pkStart, GTRecord pkEnd, List<Pair<byte[], byte[]>> selectedColumns) {
+ protected List<RawScan> prepareRawScan(GTRecord pkStart, GTRecord pkEnd, List<Pair<byte[], byte[]>> selectedColumns) {
+ List<RawScan> ret = Lists.newArrayList();
+
byte[] start = makeRowKeyToScan(pkStart, (byte) 0x00);
byte[] end = makeRowKeyToScan(pkEnd, (byte) 0xff);
//TODO fuzzy match
- return new RawScan(start, end, selectedColumns, null);
+ short cuboidShardNum = cubeSeg.getCuboidShardNum(cuboid.getId());
+
+ for (short i = 0; i < cuboidShardNum; ++i) {
+ short shard = ShardingHash.getShard(cubeSeg.getCuboidBaseShard(cuboid.getId()), i, cubeSeg.getTotalShards());
+ byte[] shardStart = Arrays.copyOf(start, start.length);
+ byte[] shardEnd = Arrays.copyOf(end, end.length);
+ BytesUtil.writeShort(shard, shardStart, 0, RowConstants.ROWKEY_SHARDID_LEN);
+ BytesUtil.writeShort(shard, shardEnd, 0, RowConstants.ROWKEY_SHARDID_LEN);
+
+ ret.add(new RawScan(shardStart, shardEnd, selectedColumns, null));
+ }
+ return ret;
+
}
private byte[] makeRowKeyToScan(GTRecord pkRec, byte fill) {
@@ -85,9 +100,12 @@ public abstract class CubeHBaseRPC {
byte[] buf = new byte[pkMaxLen + RowConstants.ROWKEY_CUBOIDID_LEN];
Arrays.fill(buf, fill);
- System.arraycopy(cuboid.getBytes(), 0, buf, 0, RowConstants.ROWKEY_CUBOIDID_LEN);
+ //for scanning/reading, later all possbile shard will be applied
+ //BytesUtil.writeShort((short) 0, buf, 0, RowConstants.ROWKEY_SHARDID_LEN);
+
+ System.arraycopy(cuboid.getBytes(), 0, buf, RowConstants.ROWKEY_SHARDID_LEN, RowConstants.ROWKEY_CUBOIDID_LEN);
if (pk != null && pk.array() != null) {
- System.arraycopy(pk.array(), pk.offset(), buf, RowConstants.ROWKEY_CUBOIDID_LEN, pk.length());
+ System.arraycopy(pk.array(), pk.offset(), buf, RowConstants.ROWKEY_HEADER_LEN, pk.length());
}
return buf;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7b9cbcbb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index e673f32..92ec59d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -20,6 +20,9 @@ import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStore;
import org.apache.kylin.storage.hbase.steps.HBaseConnection;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+
/**
* for test use only
*/
@@ -41,27 +44,40 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
final List<Pair<byte[], byte[]>> hbaseColumns = makeHBaseColumns(selectedColBlocks);
- RawScan rawScan = prepareRawScan(scanRequest.getPkStart(), scanRequest.getPkEnd(), hbaseColumns);
- Scan hbaseScan = buildScan(rawScan);
+ List<RawScan> rawScans = prepareRawScan(scanRequest.getPkStart(), scanRequest.getPkEnd(), hbaseColumns);
+
+ final List<ResultScanner> scanners = Lists.newArrayList();
+ final List<Iterator<Result>> resultIterators = Lists.newArrayList();
+
+ for (RawScan rawScan : rawScans) {
+ Scan hbaseScan = buildScan(rawScan);
+
+ final ResultScanner scanner = hbaseTable.getScanner(hbaseScan);
+ final Iterator<Result> iterator = scanner.iterator();
+
+ scanners.add(scanner);
+ resultIterators.add(iterator);
+ }
- final ResultScanner scanner = hbaseTable.getScanner(hbaseScan);
- final Iterator<Result> iterator = scanner.iterator();
+ final Iterator<Result> allResultsIterator = Iterators.concat(resultIterators.iterator());
CellListIterator cellListIterator = new CellListIterator() {
@Override
public void close() throws IOException {
- scanner.close();
+ for (ResultScanner scanner : scanners) {
+ scanner.close();
+ }
hbaseTable.close();
}
@Override
public boolean hasNext() {
- return iterator.hasNext();
+ return allResultsIterator.hasNext();
}
@Override
public List<Cell> next() {
- return iterator.next().listCells();
+ return allResultsIterator.next().listCells();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7b9cbcbb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeScanner.java
index 9359934..5e4227d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeScanner.java
@@ -201,7 +201,7 @@ public class CubeScanner implements IGTScanner {
return false;
try {
- CubeHBaseRPC rpc = new CubeHBaseEndpointRPC(cubeSeg, cuboid, info);
+ CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info);
inputScanners[cur] = rpc.getGTScanner(scanRequests.get(cur));
curIterator = inputScanners[cur].iterator();
//curIterator = new RemoteGTRecordAdapter(info, inputScanners[cur].iterator()).iterator();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7b9cbcbb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
index a6c6a23..038adae 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
@@ -96,7 +96,7 @@ public class HBaseReadonlyStore implements IGTStore {
// dimensions, set to primary key, also the 0th column block
Cell firstCell = oneRow.get(0);
- buf = byteBuffer(firstCell.getRowArray(), RowConstants.ROWKEY_CUBOIDID_LEN + firstCell.getRowOffset(), firstCell.getRowLength() - RowConstants.ROWKEY_CUBOIDID_LEN);
+ buf = byteBuffer(firstCell.getRowArray(), RowConstants.ROWKEY_HEADER_LEN + firstCell.getRowOffset(), firstCell.getRowLength() - RowConstants.ROWKEY_CUBOIDID_LEN);
oneRecord.loadCellBlock(0, buf);
// metrics
[2/2] incubator-kylin git commit: fix bug so that hbasescanrpc can
work
Posted by ma...@apache.org.
fix bug so that hbasescanrpc can work
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/62b536b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/62b536b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/62b536b0
Branch: refs/heads/KYLIN-942
Commit: 62b536b0d53e3c49139d45cdc0ccc03bd315ca61
Parents: 7b9cbcb
Author: honma <ho...@ebay.com>
Authored: Tue Sep 29 10:45:37 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Sep 29 10:45:37 2015 +0800
----------------------------------------------------------------------
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 2 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 59 +++++++++++++++++++-
.../storage/hbase/cube/v2/CubeScanner.java | 33 +----------
3 files changed, 60 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/62b536b0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 83d1fea..0a9c1d4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -97,7 +97,7 @@ public abstract class CubeHBaseRPC {
ByteArray pk = GTRecord.exportScanKey(pkRec);
int pkMaxLen = pkRec.getInfo().getMaxColumnLength(pkRec.getInfo().getPrimaryKey());
- byte[] buf = new byte[pkMaxLen + RowConstants.ROWKEY_CUBOIDID_LEN];
+ byte[] buf = new byte[pkMaxLen + RowConstants.ROWKEY_HEADER_LEN];
Arrays.fill(buf, fill);
//for scanning/reading, later all possbile shard will be applied
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/62b536b0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 92ec59d..a2ba39f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -15,6 +15,7 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStore;
@@ -28,6 +29,38 @@ import com.google.common.collect.Lists;
*/
public class CubeHBaseScanRPC extends CubeHBaseRPC {
+ static class TrimmedInfoGTRecordAdapter implements Iterable<GTRecord> {
+
+ private final GTInfo info;
+ private final Iterator<GTRecord> input;
+
+ public TrimmedInfoGTRecordAdapter(GTInfo info, Iterator<GTRecord> input) {
+ this.info = info;
+ this.input = input;
+ }
+
+ @Override
+ public Iterator<GTRecord> iterator() {
+ return new Iterator<GTRecord>() {
+ @Override
+ public boolean hasNext() {
+ return input.hasNext();
+ }
+
+ @Override
+ public GTRecord next() {
+ GTRecord x = input.next();
+ return new GTRecord(info, x.getInternal());
+ }
+
+ @Override
+ public void remove() {
+
+ }
+ };
+ }
+ }
+
public CubeHBaseScanRPC(CubeSegment cubeSeg, Cuboid cuboid, GTInfo fullGTInfo) {
super(cubeSeg, cuboid, fullGTInfo);
}
@@ -88,6 +121,30 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
IGTStore store = new HBaseReadonlyStore(cellListIterator, scanRequest, hbaseColumns);
IGTScanner rawScanner = store.scan(scanRequest);
- return scanRequest.decorateScanner(rawScanner);
+
+ final IGTScanner decorateScanner = scanRequest.decorateScanner(rawScanner);
+ final TrimmedInfoGTRecordAdapter trimmedInfoGTRecordAdapter = new TrimmedInfoGTRecordAdapter(fullGTInfo, decorateScanner.iterator());
+
+ return new IGTScanner() {
+ @Override
+ public GTInfo getInfo() {
+ return fullGTInfo;
+ }
+
+ @Override
+ public int getScannedRowCount() {
+ return decorateScanner.getScannedRowCount();
+ }
+
+ @Override
+ public void close() throws IOException {
+ decorateScanner.close();
+ }
+
+ @Override
+ public Iterator<GTRecord> iterator() {
+ return trimmedInfoGTRecordAdapter.iterator();
+ }
+ };
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/62b536b0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeScanner.java
index 5e4227d..a04c562 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeScanner.java
@@ -150,37 +150,7 @@ public class CubeScanner implements IGTScanner {
return scanner.getScannedRowCount();
}
- static class RemoteGTRecordAdapter implements Iterable<GTRecord> {
-
- private final GTInfo info;
- private final Iterator<GTRecord> input;
-
- public RemoteGTRecordAdapter(GTInfo info, Iterator<GTRecord> input) {
- this.info = info;
- this.input = input;
- }
-
- @Override
- public Iterator<GTRecord> iterator() {
- return new Iterator<GTRecord>() {
- @Override
- public boolean hasNext() {
- return input.hasNext();
- }
-
- @Override
- public GTRecord next() {
- GTRecord x = input.next();
- return new GTRecord(info, x.getInternal());
- }
-
- @Override
- public void remove() {
-
- }
- };
- }
- }
+
private class Scanner {
final IGTScanner[] inputScanners = new IGTScanner[scanRequests.size()];
@@ -204,7 +174,6 @@ public class CubeScanner implements IGTScanner {
CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info);
inputScanners[cur] = rpc.getGTScanner(scanRequests.get(cur));
curIterator = inputScanners[cur].iterator();
- //curIterator = new RemoteGTRecordAdapter(info, inputScanners[cur].iterator()).iterator();
} catch (IOException e) {
throw new RuntimeException(e);
}