You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/05/24 01:22:26 UTC
incubator-kylin git commit: KYLIN-625,
refactor IGTStore to take GTRecord as pk start/end
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8.0 ce775594a -> 7d8ae6e38
KYLIN-625, refactor IGTStore to take GTRecord as pk start/end
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7d8ae6e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7d8ae6e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7d8ae6e3
Branch: refs/heads/0.8.0
Commit: 7d8ae6e3811ca53364437e0d5ccdb08f1a68d885
Parents: ce77559
Author: Yang Li <li...@apache.org>
Authored: Sun May 24 07:22:07 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun May 24 07:22:07 2015 +0800
----------------------------------------------------------------------
.../storage/cube/CubeHBaseReadonlyStore.java | 9 ++++--
.../kylin/storage/gridtable/GTComboStore.java | 11 +++----
.../kylin/storage/gridtable/GTRawScanner.java | 7 +----
.../kylin/storage/gridtable/GTRecord.java | 25 +++++++++++++++
.../kylin/storage/gridtable/GTRowBlock.java | 2 +-
.../kylin/storage/gridtable/IGTStore.java | 2 +-
.../apache/kylin/storage/gridtable/ScanKey.java | 33 --------------------
.../gridtable/diskstore/GTDiskStore.java | 4 +--
.../gridtable/memstore/GTSimpleMemStore.java | 4 +--
9 files changed, 43 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
index cb67fc5..cd0904b 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
@@ -23,6 +23,7 @@ import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
import org.apache.kylin.cube.model.HBaseMappingDesc;
import org.apache.kylin.storage.gridtable.GTInfo;
+import org.apache.kylin.storage.gridtable.GTRecord;
import org.apache.kylin.storage.gridtable.GTRowBlock;
import org.apache.kylin.storage.gridtable.GTRowBlock.Writer;
import org.apache.kylin.storage.gridtable.GTScanRequest;
@@ -64,7 +65,7 @@ public class CubeHBaseReadonlyStore implements IGTStore {
}
@Override
- public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColumnBlocks, GTScanRequest additionalPushDown) throws IOException {
+ public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColumnBlocks, GTScanRequest additionalPushDown) throws IOException {
// TODO enable coprocessor
// primary key (also the 0th column block) is always selected
@@ -122,7 +123,7 @@ public class CubeHBaseReadonlyStore implements IGTStore {
};
}
- private Scan buildScan(ByteArray pkStart, ByteArray pkEnd, List<Pair<byte[], byte[]>> selectedColumns) {
+ private Scan buildScan(GTRecord pkStart, GTRecord pkEnd, List<Pair<byte[], byte[]>> selectedColumns) {
Scan scan = new Scan();
scan.setCaching(SCAN_CACHE);
scan.setCacheBlocks(true);
@@ -140,8 +141,10 @@ public class CubeHBaseReadonlyStore implements IGTStore {
return scan;
}
- private byte[] makeRowKeyToScan(ByteArray pk, byte fill) {
+ private byte[] makeRowKeyToScan(GTRecord pkRec, byte fill) {
+ ByteArray pk = GTRecord.exportScanKey(pkRec);
int pkMaxLen = info.getMaxColumnLength(info.getPrimaryKey());
+
byte[] buf = new byte[pkMaxLen + RowConstants.ROWKEY_CUBOIDID_LEN];
Arrays.fill(buf, fill);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
index c0d56c3..92707ed 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
@@ -1,14 +1,13 @@
package org.apache.kylin.storage.gridtable;
-import org.apache.kylin.common.util.ByteArray;
+import java.io.IOException;
+import java.util.BitSet;
+
import org.apache.kylin.storage.gridtable.diskstore.GTDiskStore;
import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.BitSet;
-
/**
*/
public class GTComboStore implements IGTStore {
@@ -18,7 +17,7 @@ public class GTComboStore implements IGTStore {
private final GTInfo gtInfo;
private void convert(IGTStore input, IGTStore output) throws IOException {
- final IGTStoreScanner scanner = input.scan(ScanKey.makeScanKey(gtInfo, new GTRecord(gtInfo)), ScanKey.makeScanKey(gtInfo, new GTRecord(gtInfo)), null, null);
+ final IGTStoreScanner scanner = input.scan(null, null, null, null);
final IGTStoreWriter writer = output.rebuild(-1);
while (scanner.hasNext()) {
writer.write(scanner.next());
@@ -101,7 +100,7 @@ public class GTComboStore implements IGTStore {
}
@Override
- public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException {
+ public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException {
return getCurrent().scan(pkStart, pkEnd, selectedColBlocks, additionalPushDown);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
index ff97cd5..1e934b5 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
@@ -5,7 +5,6 @@ import java.util.BitSet;
import java.util.Iterator;
import java.util.NoSuchElementException;
-import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.storage.gridtable.IGTStore.IGTStoreScanner;
public class GTRawScanner implements IGTScanner {
@@ -23,12 +22,8 @@ public class GTRawScanner implements IGTScanner {
public GTRawScanner(GTInfo info, IGTStore store, GTScanRequest req) throws IOException {
this.info = info;
-
- ByteArray start = ScanKey.makeScanKey(info, req.getPkStart());
- ByteArray end = ScanKey.makeScanKey(info, req.getPkEnd());
this.selectedColBlocks = info.selectColumnBlocks(req.getColumns());
-
- this.storeScanner = store.scan(start, end, selectedColBlocks, req);
+ this.storeScanner = store.scan(req.getPkStart(), req.getPkEnd(), selectedColBlocks, req);
this.oneRecord = new GTRecord(info);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
index 61ff25d..3d0f9bb 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
@@ -233,4 +233,29 @@ public class GTRecord implements Comparable<GTRecord> {
}
}
+ /* similar to export(primaryKey) but will stop at the first null value */
+ public static ByteArray exportScanKey(GTRecord rec) {
+ if (rec == null)
+ return null;
+
+ GTInfo info = rec.getInfo();
+
+ BitSet selectedColumns = new BitSet();
+ int len = 0;
+ for (int i = info.primaryKey.nextSetBit(0); i >= 0; i = info.primaryKey.nextSetBit(i + 1)) {
+ if (rec.cols[i].array() == null) {
+ break;
+ }
+ selectedColumns.set(i);
+ len += rec.cols[i].length();
+ }
+
+ if (selectedColumns.cardinality() == 0)
+ return null;
+
+ ByteArray buf = ByteArray.allocate(len);
+ rec.exportColumns(selectedColumns, buf);
+ return buf;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
index ec24da6..4a68659 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
@@ -192,7 +192,7 @@ public class GTRowBlock {
}
}
- public void export(DataOutputStream dataOutputStream, ByteArray array) throws IOException {
+ private void export(DataOutputStream dataOutputStream, ByteArray array) throws IOException {
dataOutputStream.writeInt(array.length());
dataOutputStream.write(array.array(), array.offset(), array.length());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
index 4defa24..76bf76a 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
@@ -15,7 +15,7 @@ public interface IGTStore {
IGTStoreWriter append(int shard, GTRowBlock.Writer fillLast) throws IOException;
- IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException;
+ IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException;
interface IGTStoreWriter extends Closeable {
void write(GTRowBlock block) throws IOException;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java
deleted file mode 100644
index d8edf45..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package org.apache.kylin.storage.gridtable;
-
-import org.apache.kylin.common.util.ByteArray;
-
-import java.util.BitSet;
-
-/**
- */
-public final class ScanKey {
-
- private ScanKey() {
- }
-
- static ByteArray makeScanKey(GTInfo info, GTRecord rec) {
- int firstPKCol = info.primaryKey.nextSetBit(0);
- if (rec == null || rec.cols[firstPKCol].array() == null)
- return null;
-
- BitSet selectedColumns = new BitSet();
- int len = 0;
- for (int i = info.primaryKey.nextSetBit(0); i >= 0; i = info.primaryKey.nextSetBit(i + 1)) {
- if (rec.cols[i].array() == null) {
- break;
- }
- selectedColumns.set(i);
- len += rec.cols[i].length();
- }
-
- ByteArray buf = ByteArray.allocate(len);
- rec.exportColumns(selectedColumns, buf);
- return buf;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
index 9684d8d..b6f7d95 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
@@ -10,8 +10,8 @@ import java.nio.ByteBuffer;
import java.util.BitSet;
import java.util.UUID;
-import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.storage.gridtable.GTInfo;
+import org.apache.kylin.storage.gridtable.GTRecord;
import org.apache.kylin.storage.gridtable.GTRowBlock;
import org.apache.kylin.storage.gridtable.GTScanRequest;
import org.apache.kylin.storage.gridtable.IGTStore;
@@ -151,7 +151,7 @@ public class GTDiskStore implements IGTStore {
}
@Override
- public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException {
+ public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException {
return new DiskStoreScanner(fileSystem.getReader(getRowBlockFile(identifier)));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
index c59ee04..a4d0b8d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
@@ -6,8 +6,8 @@ import java.util.BitSet;
import java.util.Iterator;
import java.util.List;
-import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.storage.gridtable.GTInfo;
+import org.apache.kylin.storage.gridtable.GTRecord;
import org.apache.kylin.storage.gridtable.GTRowBlock;
import org.apache.kylin.storage.gridtable.GTScanRequest;
import org.apache.kylin.storage.gridtable.IGTStore;
@@ -78,7 +78,7 @@ public class GTSimpleMemStore implements IGTStore {
}
@Override
- public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) {
+ public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) {
return new IGTStoreScanner() {
Iterator<GTRowBlock> it = rowBlockList.iterator();