You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/06/24 07:05:06 UTC
[28/50] kylin git commit: refactor: minor changes
refactor: minor changes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b361d8aa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b361d8aa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b361d8aa
Branch: refs/heads/stream_m1
Commit: b361d8aaa2b1b5d0d86107bedb9f34397be000a5
Parents: ea24450
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Jun 22 13:59:58 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Jun 22 14:00:15 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/kv/RowKeyEncoder.java | 21 ++++++-
.../org/apache/kylin/gridtable/GTRecord.java | 17 +-----
.../gridtable/memstore/GTSimpleMemStore.java | 20 +++++-
.../kylin/gridtable/DictGridTableTest.java | 64 ++++++++++----------
.../coprocessor/endpoint/CubeVisitService.java | 2 +-
5 files changed, 70 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
index db704dc..ebcbadd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
@@ -94,12 +94,31 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
@Override
public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] buf) {
ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0);
- record.exportColumns(keyColumns, byteArray, defaultValue());
+
+ encodeDims(record, keyColumns, byteArray, defaultValue());
//fill shard and cuboid
fillHeader(buf);
}
+ //ByteArray representing dimension does not have extra header
+ public void encodeDims(GTRecord record, ImmutableBitSet selectedCols, ByteArray buf, byte defaultValue) {
+ int pos = 0;
+ for (int i = 0; i < selectedCols.trueBitCount(); i++) {
+ int c = selectedCols.trueBitAt(i);
+ ByteArray columnC = record.get(c);
+ if (columnC.array() != null) {
+ System.arraycopy(record.get(c).array(), columnC.offset(), buf.array(), buf.offset() + pos, columnC.length());
+ pos += columnC.length();
+ } else {
+ int maxLength = record.getInfo().getCodeSystem().maxCodeLength(c);
+ Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + pos + maxLength, defaultValue);
+ pos += maxLength;
+ }
+ }
+ buf.setLength(pos);
+ }
+
@Override
public void encode(ByteArray bodyBytes, ByteArray outputBuf) {
Preconditions.checkState(bodyBytes.length() == bodyLength);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index b676693..276f66e 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -235,22 +235,7 @@ public class GTRecord implements Comparable<GTRecord> {
buf.setLength(pos);
}
- /** write data to given buffer, like serialize, use defaultValue when required column is not set*/
- public void exportColumns(ImmutableBitSet selectedCols, ByteArray buf, byte defaultValue) {
- int pos = 0;
- for (int i = 0; i < selectedCols.trueBitCount(); i++) {
- int c = selectedCols.trueBitAt(i);
- if (cols[c].array() != null) {
- System.arraycopy(cols[c].array(), cols[c].offset(), buf.array(), buf.offset() + pos, cols[c].length());
- pos += cols[c].length();
- } else {
- int maxLength = info.codeSystem.maxCodeLength(c);
- Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + pos + maxLength, defaultValue);
- pos += maxLength;
- }
- }
- buf.setLength(pos);
- }
+
/** write data to given buffer, like serialize */
public void exportColumns(ImmutableBitSet selectedCols, ByteBuffer buf) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
index a1e81a8..ec5d22d 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
@@ -25,6 +25,7 @@ import java.util.Iterator;
import java.util.List;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
@@ -34,14 +35,23 @@ import org.apache.kylin.gridtable.IGTWriter;
public class GTSimpleMemStore implements IGTStore {
- final GTInfo info;
- final List<byte[]> rowList;
+ final protected GTInfo info;
+ final protected List<byte[]> rowList;
+
+ protected GTSimpleMemStore(GTInfo info, List<byte[]> rowList) {
+ this.info = info;
+ this.rowList = rowList;
+ }
public GTSimpleMemStore(GTInfo info) {
this.info = info;
this.rowList = new ArrayList<byte[]>();
}
+ public List<byte[]> getRowList() {
+ return rowList;
+ }
+
@Override
public GTInfo getInfo() {
return info;
@@ -80,6 +90,10 @@ public class GTSimpleMemStore implements IGTStore {
}
}
+ protected ImmutableBitSet getColumns() {
+ return info.getAllColumns();
+ }
+
@Override
public IGTScanner scan(GTScanRequest scanRequest) {
@@ -115,7 +129,7 @@ public class GTSimpleMemStore implements IGTStore {
@Override
public GTRecord next() {
byte[] bytes = it.next();
- oneRecord.loadColumns(info.getAllColumns(), ByteBuffer.wrap(bytes));
+ oneRecord.loadColumns(getColumns(), ByteBuffer.wrap(bytes));
count++;
return oneRecord;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index dd2b98e..1687a4f 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -17,7 +17,7 @@
package org.apache.kylin.gridtable;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.math.BigDecimal;
@@ -60,7 +60,7 @@ import org.junit.Test;
import com.google.common.collect.Lists;
-public class DictGridTableTest extends LocalFileMetadataTestCase {
+public class DictGridTableTest extends LocalFileMetadataTestCase {
private GridTable table;
private GTInfo info;
@@ -78,15 +78,15 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
@After
public void after() throws Exception {
-
+
this.cleanupTestMetadata();
}
@Before
public void setup() throws IOException {
-
+
this.createTestMetadata();
-
+
table = newTestTable();
info = table.getInfo();
@@ -112,10 +112,9 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
ByteArray segmentEnd = enc(info, 0, "2015-01-15");
assertEquals(segmentStart, segmentStartX);
-
{
LogicalTupleFilter filter = and(timeComp0, ageComp1);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());//scan range are [close,close]
assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString());
@@ -124,25 +123,25 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
}
{
LogicalTupleFilter filter = and(timeComp2, ageComp1);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(0, r.size());
}
{
LogicalTupleFilter filter = and(timeComp4, ageComp1);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());
}
{
LogicalTupleFilter filter = and(timeComp5, ageComp1);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(0, r.size());
}
{
LogicalTupleFilter filter = or(and(timeComp2, ageComp1), and(timeComp1, ageComp1), and(timeComp6, ageComp1));
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());
assertEquals("[1421193600000, 10]-[null, 10]", r.get(0).toString());
@@ -150,7 +149,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
}
{
LogicalTupleFilter filter = or(timeComp2, timeComp1, timeComp6);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());
assertEquals("[1421193600000, null]-[null, null]", r.get(0).toString());
@@ -159,14 +158,14 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
{
//skip FALSE filter
LogicalTupleFilter filter = and(ageComp1, ConstantTupleFilter.FALSE);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(0, r.size());
}
{
//TRUE or FALSE filter
LogicalTupleFilter filter = or(ConstantTupleFilter.TRUE, ConstantTupleFilter.FALSE);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());
assertEquals("[null, null]-[null, null]", r.get(0).toString());
@@ -174,7 +173,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
{
//TRUE or other filter
LogicalTupleFilter filter = or(ageComp1, ConstantTupleFilter.TRUE);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());
assertEquals("[null, null]-[null, null]", r.get(0).toString());
@@ -187,7 +186,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
{
LogicalTupleFilter filter = and(timeComp0, ageComp1);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());//scan range are [close,close]
assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString());
@@ -197,7 +196,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
{
LogicalTupleFilter filter = and(timeComp5, ageComp1);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0),filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, Pair.newPair(new ByteArray(), segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(0, r.size());//scan range are [close,close]
}
@@ -206,11 +205,10 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
@Test
public void verifyScanRangePlanner() {
-
// flatten or-and & hbase fuzzy value
{
LogicalTupleFilter filter = and(timeComp1, or(ageComp1, ageComp2));
- GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null,filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null, filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(1, r.size());
assertEquals("[1421193600000, 10]-[null, 20]", r.get(0).toString());
@@ -220,7 +218,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
// pre-evaluate ever false
{
LogicalTupleFilter filter = and(timeComp1, timeComp2);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null,filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null, filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(0, r.size());
}
@@ -228,7 +226,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
// pre-evaluate ever true
{
LogicalTupleFilter filter = or(timeComp1, ageComp4);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null,filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null, filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals("[[null, null]-[null, null]]", r.toString());
}
@@ -236,7 +234,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
// merge overlap range
{
LogicalTupleFilter filter = or(timeComp1, timeComp3);
- GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null,filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null, filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals("[[null, null]-[null, null]]", r.toString());
}
@@ -244,7 +242,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
// merge too many ranges
{
LogicalTupleFilter filter = or(and(timeComp4, ageComp1), and(timeComp4, ageComp2), and(timeComp4, ageComp3));
- GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null,filter);
+ GTScanRangePlanner planner = new GTScanRangePlanner(info, null, null, filter);
List<GTScanRange> r = planner.planScanRanges();
assertEquals(3, r.size());
assertEquals("[1421280000000, 10]-[1421280000000, 10]", r.get(0).toString());
@@ -271,7 +269,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
}
//for testing GTScanRequest serialization and deserialization
- private GTScanRequest useDeserializedGTScanRequest(GTScanRequest origin) {
+ public static GTScanRequest useDeserializedGTScanRequest(GTScanRequest origin) {
ByteBuffer buffer = ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE);
GTScanRequest.serializer.serialize(origin, buffer);
buffer.flip();
@@ -445,38 +443,38 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
scanner.close();
}
- private ByteArray enc(GTInfo info, int col, String value) {
+ public static ByteArray enc(GTInfo info, int col, String value) {
ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength());
info.codeSystem.encodeColumnValue(col, value, buf);
return ByteArray.copyOf(buf.array(), buf.arrayOffset(), buf.position());
}
- private ExtractTupleFilter unevaluatable(TblColRef col) {
+ public static ExtractTupleFilter unevaluatable(TblColRef col) {
ExtractTupleFilter r = new ExtractTupleFilter(FilterOperatorEnum.EXTRACT);
r.addChild(new ColumnTupleFilter(col));
return r;
}
- private CompareTupleFilter compare(TblColRef col, FilterOperatorEnum op, Object... value) {
+ public static CompareTupleFilter compare(TblColRef col, FilterOperatorEnum op, Object... value) {
CompareTupleFilter result = new CompareTupleFilter(op);
result.addChild(new ColumnTupleFilter(col));
result.addChild(new ConstantTupleFilter(Arrays.asList(value)));
return result;
}
- private LogicalTupleFilter and(TupleFilter... children) {
+ public static LogicalTupleFilter and(TupleFilter... children) {
return logic(FilterOperatorEnum.AND, children);
}
- private LogicalTupleFilter or(TupleFilter... children) {
+ public static LogicalTupleFilter or(TupleFilter... children) {
return logic(FilterOperatorEnum.OR, children);
}
- private LogicalTupleFilter not(TupleFilter child) {
+ public static LogicalTupleFilter not(TupleFilter child) {
return logic(FilterOperatorEnum.NOT, child);
}
- private LogicalTupleFilter logic(FilterOperatorEnum op, TupleFilter... children) {
+ public static LogicalTupleFilter logic(FilterOperatorEnum op, TupleFilter... children) {
LogicalTupleFilter result = new LogicalTupleFilter(op);
for (TupleFilter c : children) {
result.addChild(c);
@@ -484,7 +482,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
return result;
}
- static GridTable newTestTable() throws IOException {
+ public static GridTable newTestTable() throws IOException {
GTInfo info = newInfo();
GTSimpleMemStore store = new GTSimpleMemStore(info);
GridTable table = new GridTable(info, store);
@@ -609,7 +607,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
return builder.build(0);
}
- private static ImmutableBitSet setOf(int... values) {
+ public static ImmutableBitSet setOf(int... values) {
BitSet set = new BitSet();
for (int i : values)
set.set(i);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b361d8aa/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index c3d2a0d..3827aa9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -312,7 +312,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try {
oneRecord.exportColumns(scanReq.getColumns(), buffer);
} catch (BufferOverflowException boe) {
- buffer = ByteBuffer.allocate((int) (oneRecord.sizeOf(scanReq.getColumns()) * 1.5));
+ buffer = ByteBuffer.allocate(oneRecord.sizeOf(scanReq.getColumns()) * 2);
oneRecord.exportColumns(scanReq.getColumns(), buffer);
}