You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/06/22 06:03:24 UTC
[1/3] kylin git commit: refactor: minor changes
Repository: kylin
Updated Branches:
refs/heads/master ff1c81395 -> b361d8aaa
refactor: minor changes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b361d8aa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b361d8aa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b361d8aa
Branch: refs/heads/master
Commit: b361d8aaa2b1b5d0d86107bedb9f34397be000a5
Parents: ea24450
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Jun 22 13:59:58 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Jun 22 14:00:15 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/kv/RowKeyEncoder.java | 21 ++++++-
.../org/apache/kylin/gridtable/GTRecord.java | 17 +-----
.../gridtable/memstore/GTSimpleMemStore.java | 20 +++++-
.../kylin/gridtable/DictGridTableTest.java | 64 ++++++++++----------
.../coprocessor/endpoint/CubeVisitService.java | 2 +-
5 files changed, 70 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
index db704dc..ebcbadd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
@@ -94,12 +94,31 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
@Override
public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] buf) {
ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0);
- record.exportColumns(keyColumns, byteArray, defaultValue());
+
+ encodeDims(record, keyColumns, byteArray, defaultValue());
//fill shard and cuboid
fillHeader(buf);
}
+ //ByteArray representing dimension does not have extra header
+ public void encodeDims(GTRecord record, ImmutableBitSet selectedCols, ByteArray buf, byte defaultValue) {
+ int pos = 0;
+ for (int i = 0; i < selectedCols.trueBitCount(); i++) {
+ int c = selectedCols.trueBitAt(i);
+ ByteArray columnC = record.get(c);
+ if (columnC.array() != null) {
+ System.arraycopy(record.get(c).array(), columnC.offset(), buf.array(), buf.offset() + pos, columnC.length());
+ pos += columnC.length();
+ } else {
+ int maxLength = record.getInfo().getCodeSystem().maxCodeLength(c);
+ Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + pos + maxLength, defaultValue);
+ pos += maxLength;
+ }
+ }
+ buf.setLength(pos);
+ }
+
@Override
public void encode(ByteArray bodyBytes, ByteArray outputBuf) {
Preconditions.checkState(bodyBytes.length() == bodyLength);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index b676693..276f66e 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -235,22 +235,7 @@ public class GTRecord implements Comparable<GTRecord> {
buf.setLength(pos);
}
- /** write data to given buffer, like serialize, use defaultValue when required column is not set*/
- public void exportColumns(ImmutableBitSet selectedCols, ByteArray buf, byte defaultValue) {
- int pos = 0;
- for (int i = 0; i < selectedCols.trueBitCount(); i++) {
- int c = selectedCols.trueBitAt(i);
- if (cols[c].array() != null) {
- System.arraycopy(cols[c].array(), cols[c].offset(), buf.array(), buf.offset() + pos, cols[c].length());
- pos += cols[c].length();
- } else {
- int maxLength = info.codeSystem.maxCodeLength(c);
- Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + pos + maxLength, defaultValue);
- pos += maxLength;
- }
- }
- buf.setLength(pos);
- }
+
/** write data to given buffer, like serialize */
public void exportColumns(ImmutableBitSet selectedCols, ByteBuffer buf) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
index a1e81a8..ec5d22d 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
@@ -25,6 +25,7 @@ import java.util.Iterator;
import java.util.List;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
@@ -34,14 +35,23 @@ import org.apache.kylin.gridtable.IGTWriter;
public class GTSimpleMemStore implements IGTStore {
- final GTInfo info;
- final List<byte[]> rowList;
+ final protected GTInfo info;
+ final protected List<byte[]> rowList;
+
+ protected GTSimpleMemStore(GTInfo info, List<byte[]> rowList) {
+ this.info = info;
+ this.rowList = rowList;
+ }
public GTSimpleMemStore(GTInfo info) {
this.info = info;
this.rowList = new ArrayList<byte[]>();
}
+ public List<byte[]> getRowList() {
+ return rowList;
+ }
+
@Override
public GTInfo getInfo() {
return info;
@@ -80,6 +90,10 @@ public class GTSimpleMemStore implements IGTStore {
}
}
+ protected ImmutableBitSet getColumns() {
+ return info.getAllColumns();
+ }
+
@Override
public IGTScanner scan(GTScanRequest scanRequest) {
@@ -115,7 +129,7 @@ public class GTSimpleMemStore implements IGTStore {
@Override
public GTRecord next() {
byte[] bytes = it.next();
- oneRecord.loadColumns(info.getAllColumns(), ByteBuffer.wrap(bytes));
+ oneRecord.loadColumns(getColumns(), ByteBuffer.wrap(bytes));
count++;
return oneRecord;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index dd2b98e..1687a4f 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -17,7 +17,7 @@
package org.apache.kylin.gridtable;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.math.BigDecimal;
@@ -60,7 +60,7 @@ import org.junit.Test;
import com.google.common.collect.Lists;
-public class DictGridTableTest extends LocalFileMetadataTestCase {
+public class DictGridTableTest extends LocalFileMetadataTestCase {
private GridTable table;
private GTInfo info;
@@ -78,15 +78,15 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
@After
public void after() throws Exception {
-
+
this.cleanupTestMetadata();
}
@Before
public void setup() throws IOException {
-
+
this.createTestMetadata();
-
+
table = newTestTable();
info = table.getInfo();
@@ -112,10 +112,9 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
ByteArray segmentEnd = enc(info, 0, "2015-01-15");
assertEquals(segmentStart, segmentStartX);
-
{
LogicalTupleFilter filter = and(timeComp0, ageComp1);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());//scan range are [close,close]
assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString());
@@ -124,25 +123,25 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
}
{
LogicalTupleFilter filter = and(timeComp2, ageComp1);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(0, r.size());
}
{
LogicalTupleFilter filter = and(timeComp4, ageComp1);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());
}
{
LogicalTupleFilter filter = and(timeComp5, ageComp1);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(0, r.size());
}
{
LogicalTupleFilter filter = or(and(timeComp2, ageComp1), and(timeComp1, ageComp1), and(timeComp6, ageComp1));
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());
assertEquals("[1421193600000, 10]-[null, 10]", r.get(0).toString());
@@ -150,7 +149,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
}
{
LogicalTupleFilter filter = or(timeComp2, timeComp1, timeComp6);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());
assertEquals("[1421193600000, null]-[null, null]", r.get(0).toString());
@@ -159,14 +158,14 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
{
//skip FALSE filter
LogicalTupleFilter filter = and(ageComp1, ConstantTupleFilter.FALSE);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(0, r.size());
}
{
//TRUE or FALSE filter
LogicalTupleFilter filter = or(ConstantTupleFilter.TRUE, ConstantTupleFilter.FALSE);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());
assertEquals("[null, null]-[null, null]", r.get(0).toString());
@@ -174,7 +173,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
{
//TRUE or other filter
LogicalTupleFilter filter = or(ageComp1, ConstantTupleFilter.TRUE);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());
assertEquals("[null, null]-[null, null]", r.get(0).toString());
@@ -187,7 +186,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
{
LogicalTupleFilter filter = and(timeComp0, ageComp1);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());//scan range are [close,close]
assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString());
@@ -197,7 +196,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
{
LogicalTupleFilter filter = and(timeComp5, ageComp1);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(0, r.size());//scan range are [close,close]
}
@@ -206,11 +205,10 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
@Test
public void verifyScanRangePlanner() {
-
// flatten or-and & hbase fuzzy value
{
LogicalTupleFilter filter = and(timeComp1, or(ageComp1, ageComp2));
- GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null,filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null, filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());
assertEquals("[1421193600000, 10]-[null, 20]", r.get(0).toString());
@@ -220,7 +218,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
// pre-evaluate ever false
{
LogicalTupleFilter filter = and(timeComp1, timeComp2);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null,filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null, filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(0, r.size());
}
@@ -228,7 +226,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
// pre-evaluate ever true
{
LogicalTupleFilter filter = or(timeComp1, ageComp4);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null,filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null, filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals("[[null, null]-[null, null]]", r.toString());
}
@@ -236,7 +234,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
// merge overlap range
{
LogicalTupleFilter filter = or(timeComp1, timeComp3);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null,filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null, filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals("[[null, null]-[null, null]]", r.toString());
}
@@ -244,7 +242,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
// merge too many ranges
{
LogicalTupleFilter filter = or(and(timeComp4, ageComp1), and(timeComp4, ageComp2), and(timeComp4, ageComp3));
- GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null,filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null, filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(3, r.size());
assertEquals("[1421280000000, 10]-[1421280000000, 10]", r.get(0).toString());
@@ -271,7 +269,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
}
//for testing GTScanRequest serialization and deserialization
- private GTScanRequest useDeserializedGTScanRequest(GTScanRequest origin) {
+ public static GTScanRequest useDeserializedGTScanRequest(GTScanRequest origin) {
ByteBuffer buffer = ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE);
GTScanRequest.serializer.serialize(origin, buffer);
buffer.flip();
@@ -445,38 +443,38 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
scanner.close();
}
- private ByteArray enc(GTInfo info, int col, String value) {
+ public static ByteArray enc(GTInfo info, int col, String value) {
ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength());
info.codeSystem.encodeColumnValue(col, value, buf);
return ByteArray.copyOf(buf.array(), buf.arrayOffset(), buf.position());
}
- private ExtractTupleFilter unevaluatable(TblColRef col) {
+ public static ExtractTupleFilter unevaluatable(TblColRef col) {
ExtractTupleFilter r = new ExtractTupleFilter(FilterOperatorEnum.EXTRACT);
r.addChild(new ColumnTupleFilter(col));
return r;
}
- private CompareTupleFilter compare(TblColRef col, FilterOperatorEnum op, Object... value) {
+ public static CompareTupleFilter compare(TblColRef col, FilterOperatorEnum op, Object... value) {
CompareTupleFilter result = new CompareTupleFilter(op);
result.addChild(new ColumnTupleFilter(col));
result.addChild(new ConstantTupleFilter(Arrays.asList(value)));
return result;
}
- private LogicalTupleFilter and(TupleFilter... children) {
+ public static LogicalTupleFilter and(TupleFilter... children) {
return logic(FilterOperatorEnum.AND, children);
}
- private LogicalTupleFilter or(TupleFilter... children) {
+ public static LogicalTupleFilter or(TupleFilter... children) {
return logic(FilterOperatorEnum.OR, children);
}
- private LogicalTupleFilter not(TupleFilter child) {
+ public static LogicalTupleFilter not(TupleFilter child) {
return logic(FilterOperatorEnum.NOT, child);
}
- private LogicalTupleFilter logic(FilterOperatorEnum op, TupleFilter... children) {
+ public static LogicalTupleFilter logic(FilterOperatorEnum op, TupleFilter... children) {
LogicalTupleFilter result = new LogicalTupleFilter(op);
for (TupleFilter c : children) {
result.addChild(c);
@@ -484,7 +482,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
return result;
}
- static GridTable newTestTable() throws IOException {
+ public static GridTable newTestTable() throws IOException {
GTInfo info = newInfo();
GTSimpleMemStore store = new GTSimpleMemStore(info);
GridTable table = new GridTable(info, store);
@@ -609,7 +607,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
return builder.build(0);
}
- private static ImmutableBitSet setOf(int... values) {
+ public static ImmutableBitSet setOf(int... values) {
BitSet set = new BitSet();
for (int i : values)
set.set(i);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index c3d2a0d..3827aa9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -312,7 +312,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try {
oneRecord.exportColumns(scanReq.getColumns(), buffer);
} catch (BufferOverflowException boe) {
- buffer = ByteBuffer.allocate((int) (oneRecord.sizeOf(scanReq.getColumns()) * 1.5));
+ buffer = ByteBuffer.allocate(oneRecord.sizeOf(scanReq.getColumns()) * 2);
oneRecord.exportColumns(scanReq.getColumns(), buffer);
}
[2/3] kylin git commit: refactor: remove FuzzyKeyGTRecord
Posted by ma...@apache.org.
refactor: remove FuzzyKeyGTRecord
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea244504
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea244504
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea244504
Branch: refs/heads/master
Commit: ea244504e5fa66ef14923f6665bbdbafcd8c28e2
Parents: 64b5f86
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Jun 21 14:56:50 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Jun 22 14:00:15 2016 +0800
----------------------------------------------------------------------
.../kylin/gridtable/FuzzyKeyGTRecord.java | 37 --------------------
.../org/apache/kylin/gridtable/GTScanRange.java | 12 +++----
.../kylin/gridtable/GTScanRangePlanner.java | 18 +++++-----
.../apache/kylin/gridtable/GTScanRequest.java | 18 +++-------
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 7 ++--
5 files changed, 22 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea244504/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java
deleted file mode 100644
index acffb4d..0000000
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.gridtable;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.ImmutableBitSet;
-
-public class FuzzyKeyGTRecord extends GTRecord {
- final ImmutableBitSet maskForEqualHashComp;
-
- public FuzzyKeyGTRecord(GTInfo info, ByteArray[] cols, ImmutableBitSet maskForEqualHashComp) {
- super(info, cols);
- this.maskForEqualHashComp = maskForEqualHashComp;
- }
-
- public FuzzyKeyGTRecord(GTInfo info,ImmutableBitSet maskForEqualHashComp)
- {
- super(info);
- this.maskForEqualHashComp = maskForEqualHashComp;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea244504/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
index 433626e..0cffcd9 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
@@ -27,25 +27,25 @@ public class GTScanRange {
final public GTRecord pkStart; // inclusive, record must not be null, col[pk].array() can be null to mean unbounded
final public GTRecord pkEnd; // inclusive, record must not be null, col[pk].array() can be null to mean unbounded
- final public List<FuzzyKeyGTRecord> fuzzyKeys; // partial matching primary keys
+ final public List<GTRecord> fuzzyKeys; // partial matching primary keys
public GTScanRange(GTRecord pkStart, GTRecord pkEnd) {
this(pkStart, pkEnd, null);
}
- public GTScanRange(GTRecord pkStart, GTRecord pkEnd, List<FuzzyKeyGTRecord> fuzzyKeys) {
+ public GTScanRange(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> fuzzyKeys) {
GTInfo info = pkStart.info;
assert info == pkEnd.info;
this.pkStart = pkStart;
this.pkEnd = pkEnd;
- this.fuzzyKeys = fuzzyKeys == null ? Collections.<FuzzyKeyGTRecord> emptyList() : fuzzyKeys;
+ this.fuzzyKeys = fuzzyKeys == null ? Collections.<GTRecord> emptyList() : fuzzyKeys;
}
public GTScanRange replaceGTInfo(final GTInfo gtInfo) {
- List<FuzzyKeyGTRecord> newFuzzyKeys = Lists.newArrayList();
- for (FuzzyKeyGTRecord input : fuzzyKeys) {
- newFuzzyKeys.add(new FuzzyKeyGTRecord(gtInfo, input.cols, input.maskForEqualHashComp));
+ List<GTRecord> newFuzzyKeys = Lists.newArrayList();
+ for (GTRecord input : fuzzyKeys) {
+ newFuzzyKeys.add(new GTRecord(gtInfo, input.cols));
}
return new GTScanRange(new GTRecord(gtInfo, pkStart.cols), //
new GTRecord(gtInfo, pkEnd.cols), //
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea244504/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index 2d4c2a2..4f641e9 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -305,7 +305,7 @@ public class GTScanRangePlanner {
GTRecord pkEnd = new GTRecord(gtInfo);
Map<Integer, Set<ByteArray>> fuzzyValues = Maps.newHashMap();
- List<FuzzyKeyGTRecord> fuzzyKeys;
+ List<GTRecord> fuzzyKeys;
for (ColumnRange range : andDimRanges) {
if (gtPartitionCol != null && range.column.equals(gtPartitionCol)) {
@@ -337,8 +337,8 @@ public class GTScanRangePlanner {
return new GTScanRange(pkStart, pkEnd, fuzzyKeys);
}
- private List<FuzzyKeyGTRecord> buildFuzzyKeys(Map<Integer, Set<ByteArray>> fuzzyValueSet) {
- ArrayList<FuzzyKeyGTRecord> result = Lists.newArrayList();
+ private List<GTRecord> buildFuzzyKeys(Map<Integer, Set<ByteArray>> fuzzyValueSet) {
+ ArrayList<GTRecord> result = Lists.newArrayList();
if (fuzzyValueSet.isEmpty())
return result;
@@ -353,11 +353,11 @@ public class GTScanRangePlanner {
for (Map<Integer, ByteArray> fuzzyValue : fuzzyValueCombinations) {
- BitSet bitSet = new BitSet(gtInfo.getColumnCount());
- for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) {
- bitSet.set(entry.getKey());
- }
- FuzzyKeyGTRecord fuzzy = new FuzzyKeyGTRecord(gtInfo, new ImmutableBitSet(bitSet));
+// BitSet bitSet = new BitSet(gtInfo.getColumnCount());
+// for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) {
+// bitSet.set(entry.getKey());
+// }
+ GTRecord fuzzy = new GTRecord(gtInfo);
for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) {
fuzzy.set(entry.getKey(), entry.getValue());
}
@@ -514,7 +514,7 @@ public class GTScanRangePlanner {
GTRecord start = first.pkStart;
GTRecord end = first.pkEnd;
- List<FuzzyKeyGTRecord> newFuzzyKeys = new ArrayList<FuzzyKeyGTRecord>();
+ List<GTRecord> newFuzzyKeys = new ArrayList<GTRecord>();
boolean hasNonFuzzyRange = false;
for (GTScanRange range : ranges) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea244504/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 66c0e87..55d84e6 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -261,8 +261,8 @@ public class GTScanRequest {
serializeGTRecord(range.pkStart, out);
serializeGTRecord(range.pkEnd, out);
BytesUtil.writeVInt(range.fuzzyKeys.size(), out);
- for (FuzzyKeyGTRecord f : range.fuzzyKeys) {
- serializeFuzzyKeyGTRecord(f, out);
+ for (GTRecord f : range.fuzzyKeys) {
+ serializeGTRecord(f, out);
}
}
@@ -285,10 +285,10 @@ public class GTScanRequest {
for (int rangeIdx = 0; rangeIdx < sRangesCount; rangeIdx++) {
GTRecord sPkStart = deserializeGTRecord(in, sInfo);
GTRecord sPkEnd = deserializeGTRecord(in, sInfo);
- List<FuzzyKeyGTRecord> sFuzzyKeys = Lists.newArrayList();
+ List<GTRecord> sFuzzyKeys = Lists.newArrayList();
int sFuzzyKeySize = BytesUtil.readVInt(in);
for (int i = 0; i < sFuzzyKeySize; i++) {
- sFuzzyKeys.add(deserializeFuzzyKeyGTRecord(in, sInfo));
+ sFuzzyKeys.add(deserializeGTRecord(in, sInfo));
}
GTScanRange sRange = new GTScanRange(sPkStart, sPkEnd, sFuzzyKeys);
sRanges.add(sRange);
@@ -322,16 +322,6 @@ public class GTScanRequest {
return new GTRecord(sInfo, sCols);
}
- private void serializeFuzzyKeyGTRecord(FuzzyKeyGTRecord gtRecord, ByteBuffer out) {
- serializeGTRecord(gtRecord,out);
- ImmutableBitSet.serializer.serialize(gtRecord.maskForEqualHashComp, out);
- }
-
- private FuzzyKeyGTRecord deserializeFuzzyKeyGTRecord(ByteBuffer in, GTInfo sInfo) {
- GTRecord temp = deserializeGTRecord(in,sInfo);
- ImmutableBitSet sMaskForEqualHashComp = ImmutableBitSet.serializer.deserialize(in);
- return new FuzzyKeyGTRecord(temp.info,temp.cols, sMaskForEqualHashComp);
- }
};
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea244504/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 015edc6..af5d4b7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -40,7 +40,6 @@ import org.apache.kylin.cube.kv.RowKeyEncoder;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
import org.apache.kylin.cube.model.HBaseMappingDesc;
-import org.apache.kylin.gridtable.FuzzyKeyGTRecord;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRange;
@@ -93,7 +92,7 @@ public abstract class CubeHBaseRPC implements IGTStorage {
return scan;
}
- private RawScan preparedHBaseScan(GTRecord pkStart, GTRecord pkEnd, List<FuzzyKeyGTRecord> fuzzyKeys, ImmutableBitSet selectedColBlocks) {
+ private RawScan preparedHBaseScan(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> fuzzyKeys, ImmutableBitSet selectedColBlocks) {
final List<Pair<byte[], byte[]>> selectedColumns = makeHBaseColumns(selectedColBlocks);
LazyRowKeyEncoder encoder = new LazyRowKeyEncoder(cubeSeg, cuboid);
@@ -137,13 +136,13 @@ public abstract class CubeHBaseRPC implements IGTStorage {
* translate GTRecord format fuzzy keys to hbase expected format
* @return
*/
- private List<Pair<byte[], byte[]>> translateFuzzyKeys(List<FuzzyKeyGTRecord> fuzzyKeys) {
+ private List<Pair<byte[], byte[]>> translateFuzzyKeys(List<GTRecord> fuzzyKeys) {
if (fuzzyKeys == null || fuzzyKeys.isEmpty()) {
return Collections.emptyList();
}
List<Pair<byte[], byte[]>> ret = Lists.newArrayList();
- for (FuzzyKeyGTRecord gtRecordFuzzyKey : fuzzyKeys) {
+ for (GTRecord gtRecordFuzzyKey : fuzzyKeys) {
byte[] hbaseFuzzyKey = fuzzyKeyEncoder.createBuf();
byte[] hbaseFuzzyMask = fuzzyMaskEncoder.createBuf();
[3/3] kylin git commit: refactor: remove maskForEqualHashComp from
GTRecord
Posted by ma...@apache.org.
refactor: remove maskForEqualHashComp from GTRecord
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64b5f86f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64b5f86f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64b5f86f
Branch: refs/heads/master
Commit: 64b5f86fec6fce643cbd0f0f5a966ddc919dfbed
Parents: ff1c813
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Jun 21 14:50:27 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Jun 22 14:00:15 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/BasicTest.java | 3 +
.../kylin/gridtable/FuzzyKeyGTRecord.java | 37 ++++++++++++
.../org/apache/kylin/gridtable/GTRecord.java | 60 ++++++--------------
.../org/apache/kylin/gridtable/GTScanRange.java | 16 +++---
.../kylin/gridtable/GTScanRangePlanner.java | 16 +++---
.../apache/kylin/gridtable/GTScanRequest.java | 21 +++++--
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 7 ++-
7 files changed, 91 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index f1f5aa4..81afafe 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -79,6 +79,9 @@ public class BasicTest {
@Test
public void testxx() throws InterruptedException {
+ byte[] space = new byte[100];
+ ByteBuffer buffer = ByteBuffer.wrap(space, 10, 20);
+ buffer.put((byte) 1);
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java
new file mode 100644
index 0000000..acffb4d
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.gridtable;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
+
+public class FuzzyKeyGTRecord extends GTRecord {
+ final ImmutableBitSet maskForEqualHashComp;
+
+ public FuzzyKeyGTRecord(GTInfo info, ByteArray[] cols, ImmutableBitSet maskForEqualHashComp) {
+ super(info, cols);
+ this.maskForEqualHashComp = maskForEqualHashComp;
+ }
+
+ public FuzzyKeyGTRecord(GTInfo info,ImmutableBitSet maskForEqualHashComp)
+ {
+ super(info);
+ this.maskForEqualHashComp = maskForEqualHashComp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index f3cfc6a..b676693 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -31,39 +31,27 @@ public class GTRecord implements Comparable<GTRecord> {
final transient GTInfo info;
final ByteArray[] cols;
- final ImmutableBitSet maskForEqualHashComp;
- public GTRecord(GTInfo info, ImmutableBitSet maskForEqualHashComp, ByteArray[] cols) {
+ public GTRecord(GTInfo info, ByteArray[] cols) {
this.info = info;
this.cols = cols;
- this.maskForEqualHashComp = maskForEqualHashComp;
}
- public GTRecord(GTInfo info, ImmutableBitSet maskForEqualHashComp) {
+ public GTRecord(GTInfo info) {
this.cols = new ByteArray[info.getColumnCount()];
for (int i = 0; i < this.cols.length; i++) {
// consider column projection by pass in another bit set
this.cols[i] = new ByteArray();
}
this.info = info;
- this.maskForEqualHashComp = maskForEqualHashComp;
- }
-
- public GTRecord(GTInfo info) {
- this(info, info.colAll);
- }
-
- public GTRecord(GTInfo info, ByteArray[] cols) {
- this(info, info.colAll, cols);
}
public GTRecord(GTRecord other) {
this.info = other.info;
- this.maskForEqualHashComp = other.maskForEqualHashComp;
this.cols = new ByteArray[info.getColumnCount()];
for (int i = 0; i < other.cols.length; i++) {
this.cols[i] = other.cols[i].copy();
- }
+ }
}
public GTInfo getInfo() {
@@ -124,6 +112,7 @@ public class GTRecord implements Comparable<GTRecord> {
return result;
}
+ /** decode and return the values of this record */
public Object[] getValues(int[] selectedColumns, Object[] result) {
assert selectedColumns.length <= result.length;
for (int i = 0; i < selectedColumns.length; i++) {
@@ -136,7 +125,7 @@ public class GTRecord implements Comparable<GTRecord> {
}
return result;
}
-
+
public int sizeOf(ImmutableBitSet selectedCols) {
int size = 0;
for (int i = 0; i < selectedCols.trueBitCount(); i++) {
@@ -151,15 +140,10 @@ public class GTRecord implements Comparable<GTRecord> {
}
public GTRecord copy(ImmutableBitSet selectedCols) {
- int len = 0;
- for (int i = 0; i < selectedCols.trueBitCount(); i++) {
- int c = selectedCols.trueBitAt(i);
- len += cols[c].length();
- }
-
+ int len = sizeOf(selectedCols);
byte[] space = new byte[len];
- GTRecord copy = new GTRecord(info, this.maskForEqualHashComp);
+ GTRecord copy = new GTRecord(info);
int pos = 0;
for (int i = 0; i < selectedCols.trueBitCount(); i++) {
int c = selectedCols.trueBitAt(i);
@@ -171,10 +155,6 @@ public class GTRecord implements Comparable<GTRecord> {
return copy;
}
- public ImmutableBitSet maskForEqualHashComp() {
- return maskForEqualHashComp;
- }
-
@Override
public boolean equals(Object obj) {
if (this == obj)
@@ -187,10 +167,9 @@ public class GTRecord implements Comparable<GTRecord> {
GTRecord o = (GTRecord) obj;
if (this.info != o.info)
return false;
- if (this.maskForEqualHashComp != o.maskForEqualHashComp)
- return false;
- for (int i = 0; i < maskForEqualHashComp.trueBitCount(); i++) {
- int c = maskForEqualHashComp.trueBitAt(i);
+
+ for (int i = 0; i < info.colAll.trueBitCount(); i++) {
+ int c = info.colAll.trueBitAt(i);
if (!this.cols[c].equals(o.cols[c])) {
return false;
}
@@ -201,8 +180,8 @@ public class GTRecord implements Comparable<GTRecord> {
@Override
public int hashCode() {
int hash = 1;
- for (int i = 0; i < maskForEqualHashComp.trueBitCount(); i++) {
- int c = maskForEqualHashComp.trueBitAt(i);
+ for (int i = 0; i < info.colAll.trueBitCount(); i++) {
+ int c = info.colAll.trueBitAt(i);
hash = (31 * hash) + cols[c].hashCode();
}
return hash;
@@ -210,13 +189,12 @@ public class GTRecord implements Comparable<GTRecord> {
@Override
public int compareTo(GTRecord o) {
- assert this.info == o.info;
- assert this.maskForEqualHashComp == o.maskForEqualHashComp; // reference equal for performance
+ assert this.info == o.info; // reference equal for performance
IGTComparator comparator = info.codeSystem.getComparator();
int comp = 0;
- for (int i = 0; i < maskForEqualHashComp.trueBitCount(); i++) {
- int c = maskForEqualHashComp.trueBitAt(i);
+ for (int i = 0; i < info.colAll.trueBitCount(); i++) {
+ int c = info.colAll.trueBitAt(i);
comp = comparator.compare(cols[c], o.cols[c]);
if (comp != 0)
return comp;
@@ -226,7 +204,7 @@ public class GTRecord implements Comparable<GTRecord> {
@Override
public String toString() {
- return toString(maskForEqualHashComp);
+ return toString(info.colAll);
}
public String toString(ImmutableBitSet selectedColumns) {
@@ -238,11 +216,7 @@ public class GTRecord implements Comparable<GTRecord> {
// ============================================================================
public ByteArray exportColumns(ImmutableBitSet selectedCols) {
- int len = 0;
- for (int i = 0; i < selectedCols.trueBitCount(); i++) {
- int c = selectedCols.trueBitAt(i);
- len += cols[c].length();
- }
+ int len = sizeOf(selectedCols);
ByteArray buf = ByteArray.allocate(len);
exportColumns(selectedCols, buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
index e1b38dc..433626e 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
@@ -27,28 +27,28 @@ public class GTScanRange {
final public GTRecord pkStart; // inclusive, record must not be null, col[pk].array() can be null to mean unbounded
final public GTRecord pkEnd; // inclusive, record must not be null, col[pk].array() can be null to mean unbounded
- final public List<GTRecord> fuzzyKeys; // partial matching primary keys
+ final public List<FuzzyKeyGTRecord> fuzzyKeys; // partial matching primary keys
public GTScanRange(GTRecord pkStart, GTRecord pkEnd) {
this(pkStart, pkEnd, null);
}
- public GTScanRange(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> fuzzyKeys) {
+ public GTScanRange(GTRecord pkStart, GTRecord pkEnd, List<FuzzyKeyGTRecord> fuzzyKeys) {
GTInfo info = pkStart.info;
assert info == pkEnd.info;
this.pkStart = pkStart;
this.pkEnd = pkEnd;
- this.fuzzyKeys = fuzzyKeys == null ? Collections.<GTRecord> emptyList() : fuzzyKeys;
+ this.fuzzyKeys = fuzzyKeys == null ? Collections.<FuzzyKeyGTRecord> emptyList() : fuzzyKeys;
}
public GTScanRange replaceGTInfo(final GTInfo gtInfo) {
- List<GTRecord> newFuzzyKeys = Lists.newArrayList();
- for (GTRecord input : fuzzyKeys) {
- newFuzzyKeys.add(new GTRecord(gtInfo, input.maskForEqualHashComp(), input.cols));
+ List<FuzzyKeyGTRecord> newFuzzyKeys = Lists.newArrayList();
+ for (FuzzyKeyGTRecord input : fuzzyKeys) {
+ newFuzzyKeys.add(new FuzzyKeyGTRecord(gtInfo, input.cols, input.maskForEqualHashComp));
}
- return new GTScanRange(new GTRecord(gtInfo, pkStart.maskForEqualHashComp(), pkStart.cols), //
- new GTRecord(gtInfo, pkEnd.maskForEqualHashComp(), pkEnd.cols), //
+ return new GTScanRange(new GTRecord(gtInfo, pkStart.cols), //
+ new GTRecord(gtInfo, pkEnd.cols), //
newFuzzyKeys);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index 7d31bf7..2d4c2a2 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -305,7 +305,7 @@ public class GTScanRangePlanner {
GTRecord pkEnd = new GTRecord(gtInfo);
Map<Integer, Set<ByteArray>> fuzzyValues = Maps.newHashMap();
- List<GTRecord> fuzzyKeys;
+ List<FuzzyKeyGTRecord> fuzzyKeys;
for (ColumnRange range : andDimRanges) {
if (gtPartitionCol != null && range.column.equals(gtPartitionCol)) {
@@ -337,8 +337,8 @@ public class GTScanRangePlanner {
return new GTScanRange(pkStart, pkEnd, fuzzyKeys);
}
- private List<GTRecord> buildFuzzyKeys(Map<Integer, Set<ByteArray>> fuzzyValueSet) {
- ArrayList<GTRecord> result = Lists.newArrayList();
+ private List<FuzzyKeyGTRecord> buildFuzzyKeys(Map<Integer, Set<ByteArray>> fuzzyValueSet) {
+ ArrayList<FuzzyKeyGTRecord> result = Lists.newArrayList();
if (fuzzyValueSet.isEmpty())
return result;
@@ -357,7 +357,7 @@ public class GTScanRangePlanner {
for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) {
bitSet.set(entry.getKey());
}
- GTRecord fuzzy = new GTRecord(gtInfo, new ImmutableBitSet(bitSet));
+ FuzzyKeyGTRecord fuzzy = new FuzzyKeyGTRecord(gtInfo, new ImmutableBitSet(bitSet));
for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) {
fuzzy.set(entry.getKey(), entry.getValue());
}
@@ -514,7 +514,7 @@ public class GTScanRangePlanner {
GTRecord start = first.pkStart;
GTRecord end = first.pkEnd;
- List<GTRecord> newFuzzyKeys = new ArrayList<GTRecord>();
+ List<FuzzyKeyGTRecord> newFuzzyKeys = new ArrayList<FuzzyKeyGTRecord>();
boolean hasNonFuzzyRange = false;
for (GTScanRange range : ranges) {
@@ -774,12 +774,10 @@ public class GTScanRangePlanner {
@Override
public int compare(GTRecord a, GTRecord b) {
assert a.info == b.info;
- assert a.maskForEqualHashComp() == b.maskForEqualHashComp();
- ImmutableBitSet mask = a.maskForEqualHashComp();
int comp;
- for (int i = 0; i < mask.trueBitCount(); i++) {
- int c = mask.trueBitAt(i);
+ for (int i = 0; i < a.info.colAll.trueBitCount(); i++) {
+ int c = a.info.colAll.trueBitAt(i);
comp = comparator.compare(a.cols[c], b.cols[c]);
if (comp != 0)
return comp;
http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 0d2da43..66c0e87 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -261,8 +261,8 @@ public class GTScanRequest {
serializeGTRecord(range.pkStart, out);
serializeGTRecord(range.pkEnd, out);
BytesUtil.writeVInt(range.fuzzyKeys.size(), out);
- for (GTRecord f : range.fuzzyKeys) {
- serializeGTRecord(f, out);
+ for (FuzzyKeyGTRecord f : range.fuzzyKeys) {
+ serializeFuzzyKeyGTRecord(f, out);
}
}
@@ -285,10 +285,10 @@ public class GTScanRequest {
for (int rangeIdx = 0; rangeIdx < sRangesCount; rangeIdx++) {
GTRecord sPkStart = deserializeGTRecord(in, sInfo);
GTRecord sPkEnd = deserializeGTRecord(in, sInfo);
- List<GTRecord> sFuzzyKeys = Lists.newArrayList();
+ List<FuzzyKeyGTRecord> sFuzzyKeys = Lists.newArrayList();
int sFuzzyKeySize = BytesUtil.readVInt(in);
for (int i = 0; i < sFuzzyKeySize; i++) {
- sFuzzyKeys.add(deserializeGTRecord(in, sInfo));
+ sFuzzyKeys.add(deserializeFuzzyKeyGTRecord(in, sInfo));
}
GTScanRange sRange = new GTScanRange(sPkStart, sPkEnd, sFuzzyKeys);
sRanges.add(sRange);
@@ -311,7 +311,6 @@ public class GTScanRequest {
for (ByteArray col : gtRecord.cols) {
col.exportData(out);
}
- ImmutableBitSet.serializer.serialize(gtRecord.maskForEqualHashComp, out);
}
private GTRecord deserializeGTRecord(ByteBuffer in, GTInfo sInfo) {
@@ -320,8 +319,18 @@ public class GTScanRequest {
for (int i = 0; i < colLength; i++) {
sCols[i] = ByteArray.importData(in);
}
+ return new GTRecord(sInfo, sCols);
+ }
+
+ private void serializeFuzzyKeyGTRecord(FuzzyKeyGTRecord gtRecord, ByteBuffer out) {
+ serializeGTRecord(gtRecord,out);
+ ImmutableBitSet.serializer.serialize(gtRecord.maskForEqualHashComp, out);
+ }
+
+ private FuzzyKeyGTRecord deserializeFuzzyKeyGTRecord(ByteBuffer in, GTInfo sInfo) {
+ GTRecord temp = deserializeGTRecord(in,sInfo);
ImmutableBitSet sMaskForEqualHashComp = ImmutableBitSet.serializer.deserialize(in);
- return new GTRecord(sInfo, sMaskForEqualHashComp, sCols);
+ return new FuzzyKeyGTRecord(temp.info,temp.cols, sMaskForEqualHashComp);
}
};
http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index af5d4b7..015edc6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -40,6 +40,7 @@ import org.apache.kylin.cube.kv.RowKeyEncoder;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
import org.apache.kylin.cube.model.HBaseMappingDesc;
+import org.apache.kylin.gridtable.FuzzyKeyGTRecord;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRange;
@@ -92,7 +93,7 @@ public abstract class CubeHBaseRPC implements IGTStorage {
return scan;
}
- private RawScan preparedHBaseScan(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> fuzzyKeys, ImmutableBitSet selectedColBlocks) {
+ private RawScan preparedHBaseScan(GTRecord pkStart, GTRecord pkEnd, List<FuzzyKeyGTRecord> fuzzyKeys, ImmutableBitSet selectedColBlocks) {
final List<Pair<byte[], byte[]>> selectedColumns = makeHBaseColumns(selectedColBlocks);
LazyRowKeyEncoder encoder = new LazyRowKeyEncoder(cubeSeg, cuboid);
@@ -136,13 +137,13 @@ public abstract class CubeHBaseRPC implements IGTStorage {
* translate GTRecord format fuzzy keys to hbase expected format
* @return
*/
- private List<Pair<byte[], byte[]>> translateFuzzyKeys(List<GTRecord> fuzzyKeys) {
+ private List<Pair<byte[], byte[]>> translateFuzzyKeys(List<FuzzyKeyGTRecord> fuzzyKeys) {
if (fuzzyKeys == null || fuzzyKeys.isEmpty()) {
return Collections.emptyList();
}
List<Pair<byte[], byte[]>> ret = Lists.newArrayList();
- for (GTRecord gtRecordFuzzyKey : fuzzyKeys) {
+ for (FuzzyKeyGTRecord gtRecordFuzzyKey : fuzzyKeys) {
byte[] hbaseFuzzyKey = fuzzyKeyEncoder.createBuf();
byte[] hbaseFuzzyMask = fuzzyMaskEncoder.createBuf();