You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/06/22 06:03:26 UTC
[3/3] kylin git commit: refactor: remove maskForEqualHashComp from
GTRecord
refactor: remove maskForEqualHashComp from GTRecord
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64b5f86f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64b5f86f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64b5f86f
Branch: refs/heads/master
Commit: 64b5f86fec6fce643cbd0f0f5a966ddc919dfbed
Parents: ff1c813
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Jun 21 14:50:27 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Jun 22 14:00:15 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/BasicTest.java | 3 +
.../kylin/gridtable/FuzzyKeyGTRecord.java | 37 ++++++++++++
.../org/apache/kylin/gridtable/GTRecord.java | 60 ++++++--------------
.../org/apache/kylin/gridtable/GTScanRange.java | 16 +++---
.../kylin/gridtable/GTScanRangePlanner.java | 16 +++---
.../apache/kylin/gridtable/GTScanRequest.java | 21 +++++--
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 7 ++-
7 files changed, 91 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index f1f5aa4..81afafe 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -79,6 +79,9 @@ public class BasicTest {
@Test
public void testxx() throws InterruptedException {
+ byte[] space = new byte[100];
+ ByteBuffer buffer = ByteBuffer.wrap(space, 10, 20);
+ buffer.put((byte) 1);
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java
new file mode 100644
index 0000000..acffb4d
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/FuzzyKeyGTRecord.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.gridtable;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
+
+public class FuzzyKeyGTRecord extends GTRecord {
+ final ImmutableBitSet maskForEqualHashComp;
+
+ public FuzzyKeyGTRecord(GTInfo info, ByteArray[] cols, ImmutableBitSet maskForEqualHashComp) {
+ super(info, cols);
+ this.maskForEqualHashComp = maskForEqualHashComp;
+ }
+
+ public FuzzyKeyGTRecord(GTInfo info,ImmutableBitSet maskForEqualHashComp)
+ {
+ super(info);
+ this.maskForEqualHashComp = maskForEqualHashComp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index f3cfc6a..b676693 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -31,39 +31,27 @@ public class GTRecord implements Comparable<GTRecord> {
final transient GTInfo info;
final ByteArray[] cols;
- final ImmutableBitSet maskForEqualHashComp;
- public GTRecord(GTInfo info, ImmutableBitSet maskForEqualHashComp, ByteArray[] cols) {
+ public GTRecord(GTInfo info, ByteArray[] cols) {
this.info = info;
this.cols = cols;
- this.maskForEqualHashComp = maskForEqualHashComp;
}
- public GTRecord(GTInfo info, ImmutableBitSet maskForEqualHashComp) {
+ public GTRecord(GTInfo info) {
this.cols = new ByteArray[info.getColumnCount()];
for (int i = 0; i < this.cols.length; i++) {
// consider column projection by pass in another bit set
this.cols[i] = new ByteArray();
}
this.info = info;
- this.maskForEqualHashComp = maskForEqualHashComp;
- }
-
- public GTRecord(GTInfo info) {
- this(info, info.colAll);
- }
-
- public GTRecord(GTInfo info, ByteArray[] cols) {
- this(info, info.colAll, cols);
}
public GTRecord(GTRecord other) {
this.info = other.info;
- this.maskForEqualHashComp = other.maskForEqualHashComp;
this.cols = new ByteArray[info.getColumnCount()];
for (int i = 0; i < other.cols.length; i++) {
this.cols[i] = other.cols[i].copy();
- }
+ }
}
public GTInfo getInfo() {
@@ -124,6 +112,7 @@ public class GTRecord implements Comparable<GTRecord> {
return result;
}
+ /** decode and return the values of this record */
public Object[] getValues(int[] selectedColumns, Object[] result) {
assert selectedColumns.length <= result.length;
for (int i = 0; i < selectedColumns.length; i++) {
@@ -136,7 +125,7 @@ public class GTRecord implements Comparable<GTRecord> {
}
return result;
}
-
+
public int sizeOf(ImmutableBitSet selectedCols) {
int size = 0;
for (int i = 0; i < selectedCols.trueBitCount(); i++) {
@@ -151,15 +140,10 @@ public class GTRecord implements Comparable<GTRecord> {
}
public GTRecord copy(ImmutableBitSet selectedCols) {
- int len = 0;
- for (int i = 0; i < selectedCols.trueBitCount(); i++) {
- int c = selectedCols.trueBitAt(i);
- len += cols[c].length();
- }
-
+ int len = sizeOf(selectedCols);
byte[] space = new byte[len];
- GTRecord copy = new GTRecord(info, this.maskForEqualHashComp);
+ GTRecord copy = new GTRecord(info);
int pos = 0;
for (int i = 0; i < selectedCols.trueBitCount(); i++) {
int c = selectedCols.trueBitAt(i);
@@ -171,10 +155,6 @@ public class GTRecord implements Comparable<GTRecord> {
return copy;
}
- public ImmutableBitSet maskForEqualHashComp() {
- return maskForEqualHashComp;
- }
-
@Override
public boolean equals(Object obj) {
if (this == obj)
@@ -187,10 +167,9 @@ public class GTRecord implements Comparable<GTRecord> {
GTRecord o = (GTRecord) obj;
if (this.info != o.info)
return false;
- if (this.maskForEqualHashComp != o.maskForEqualHashComp)
- return false;
- for (int i = 0; i < maskForEqualHashComp.trueBitCount(); i++) {
- int c = maskForEqualHashComp.trueBitAt(i);
+
+ for (int i = 0; i < info.colAll.trueBitCount(); i++) {
+ int c = info.colAll.trueBitAt(i);
if (!this.cols[c].equals(o.cols[c])) {
return false;
}
@@ -201,8 +180,8 @@ public class GTRecord implements Comparable<GTRecord> {
@Override
public int hashCode() {
int hash = 1;
- for (int i = 0; i < maskForEqualHashComp.trueBitCount(); i++) {
- int c = maskForEqualHashComp.trueBitAt(i);
+ for (int i = 0; i < info.colAll.trueBitCount(); i++) {
+ int c = info.colAll.trueBitAt(i);
hash = (31 * hash) + cols[c].hashCode();
}
return hash;
@@ -210,13 +189,12 @@ public class GTRecord implements Comparable<GTRecord> {
@Override
public int compareTo(GTRecord o) {
- assert this.info == o.info;
- assert this.maskForEqualHashComp == o.maskForEqualHashComp; // reference equal for performance
+ assert this.info == o.info; // reference equal for performance
IGTComparator comparator = info.codeSystem.getComparator();
int comp = 0;
- for (int i = 0; i < maskForEqualHashComp.trueBitCount(); i++) {
- int c = maskForEqualHashComp.trueBitAt(i);
+ for (int i = 0; i < info.colAll.trueBitCount(); i++) {
+ int c = info.colAll.trueBitAt(i);
comp = comparator.compare(cols[c], o.cols[c]);
if (comp != 0)
return comp;
@@ -226,7 +204,7 @@ public class GTRecord implements Comparable<GTRecord> {
@Override
public String toString() {
- return toString(maskForEqualHashComp);
+ return toString(info.colAll);
}
public String toString(ImmutableBitSet selectedColumns) {
@@ -238,11 +216,7 @@ public class GTRecord implements Comparable<GTRecord> {
// ============================================================================
public ByteArray exportColumns(ImmutableBitSet selectedCols) {
- int len = 0;
- for (int i = 0; i < selectedCols.trueBitCount(); i++) {
- int c = selectedCols.trueBitAt(i);
- len += cols[c].length();
- }
+ int len = sizeOf(selectedCols);
ByteArray buf = ByteArray.allocate(len);
exportColumns(selectedCols, buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
index e1b38dc..433626e 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
@@ -27,28 +27,28 @@ public class GTScanRange {
final public GTRecord pkStart; // inclusive, record must not be null, col[pk].array() can be null to mean unbounded
final public GTRecord pkEnd; // inclusive, record must not be null, col[pk].array() can be null to mean unbounded
- final public List<GTRecord> fuzzyKeys; // partial matching primary keys
+ final public List<FuzzyKeyGTRecord> fuzzyKeys; // partial matching primary keys
public GTScanRange(GTRecord pkStart, GTRecord pkEnd) {
this(pkStart, pkEnd, null);
}
- public GTScanRange(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> fuzzyKeys) {
+ public GTScanRange(GTRecord pkStart, GTRecord pkEnd, List<FuzzyKeyGTRecord> fuzzyKeys) {
GTInfo info = pkStart.info;
assert info == pkEnd.info;
this.pkStart = pkStart;
this.pkEnd = pkEnd;
- this.fuzzyKeys = fuzzyKeys == null ? Collections.<GTRecord> emptyList() : fuzzyKeys;
+ this.fuzzyKeys = fuzzyKeys == null ? Collections.<FuzzyKeyGTRecord> emptyList() : fuzzyKeys;
}
public GTScanRange replaceGTInfo(final GTInfo gtInfo) {
- List<GTRecord> newFuzzyKeys = Lists.newArrayList();
- for (GTRecord input : fuzzyKeys) {
- newFuzzyKeys.add(new GTRecord(gtInfo, input.maskForEqualHashComp(), input.cols));
+ List<FuzzyKeyGTRecord> newFuzzyKeys = Lists.newArrayList();
+ for (FuzzyKeyGTRecord input : fuzzyKeys) {
+ newFuzzyKeys.add(new FuzzyKeyGTRecord(gtInfo, input.cols, input.maskForEqualHashComp));
}
- return new GTScanRange(new GTRecord(gtInfo, pkStart.maskForEqualHashComp(), pkStart.cols), //
- new GTRecord(gtInfo, pkEnd.maskForEqualHashComp(), pkEnd.cols), //
+ return new GTScanRange(new GTRecord(gtInfo, pkStart.cols), //
+ new GTRecord(gtInfo, pkEnd.cols), //
newFuzzyKeys);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index 7d31bf7..2d4c2a2 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -305,7 +305,7 @@ public class GTScanRangePlanner {
GTRecord pkEnd = new GTRecord(gtInfo);
Map<Integer, Set<ByteArray>> fuzzyValues = Maps.newHashMap();
- List<GTRecord> fuzzyKeys;
+ List<FuzzyKeyGTRecord> fuzzyKeys;
for (ColumnRange range : andDimRanges) {
if (gtPartitionCol != null && range.column.equals(gtPartitionCol)) {
@@ -337,8 +337,8 @@ public class GTScanRangePlanner {
return new GTScanRange(pkStart, pkEnd, fuzzyKeys);
}
- private List<GTRecord> buildFuzzyKeys(Map<Integer, Set<ByteArray>> fuzzyValueSet) {
- ArrayList<GTRecord> result = Lists.newArrayList();
+ private List<FuzzyKeyGTRecord> buildFuzzyKeys(Map<Integer, Set<ByteArray>> fuzzyValueSet) {
+ ArrayList<FuzzyKeyGTRecord> result = Lists.newArrayList();
if (fuzzyValueSet.isEmpty())
return result;
@@ -357,7 +357,7 @@ public class GTScanRangePlanner {
for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) {
bitSet.set(entry.getKey());
}
- GTRecord fuzzy = new GTRecord(gtInfo, new ImmutableBitSet(bitSet));
+ FuzzyKeyGTRecord fuzzy = new FuzzyKeyGTRecord(gtInfo, new ImmutableBitSet(bitSet));
for (Map.Entry<Integer, ByteArray> entry : fuzzyValue.entrySet()) {
fuzzy.set(entry.getKey(), entry.getValue());
}
@@ -514,7 +514,7 @@ public class GTScanRangePlanner {
GTRecord start = first.pkStart;
GTRecord end = first.pkEnd;
- List<GTRecord> newFuzzyKeys = new ArrayList<GTRecord>();
+ List<FuzzyKeyGTRecord> newFuzzyKeys = new ArrayList<FuzzyKeyGTRecord>();
boolean hasNonFuzzyRange = false;
for (GTScanRange range : ranges) {
@@ -774,12 +774,10 @@ public class GTScanRangePlanner {
@Override
public int compare(GTRecord a, GTRecord b) {
assert a.info == b.info;
- assert a.maskForEqualHashComp() == b.maskForEqualHashComp();
- ImmutableBitSet mask = a.maskForEqualHashComp();
int comp;
- for (int i = 0; i < mask.trueBitCount(); i++) {
- int c = mask.trueBitAt(i);
+ for (int i = 0; i < a.info.colAll.trueBitCount(); i++) {
+ int c = a.info.colAll.trueBitAt(i);
comp = comparator.compare(a.cols[c], b.cols[c]);
if (comp != 0)
return comp;
http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 0d2da43..66c0e87 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -261,8 +261,8 @@ public class GTScanRequest {
serializeGTRecord(range.pkStart, out);
serializeGTRecord(range.pkEnd, out);
BytesUtil.writeVInt(range.fuzzyKeys.size(), out);
- for (GTRecord f : range.fuzzyKeys) {
- serializeGTRecord(f, out);
+ for (FuzzyKeyGTRecord f : range.fuzzyKeys) {
+ serializeFuzzyKeyGTRecord(f, out);
}
}
@@ -285,10 +285,10 @@ public class GTScanRequest {
for (int rangeIdx = 0; rangeIdx < sRangesCount; rangeIdx++) {
GTRecord sPkStart = deserializeGTRecord(in, sInfo);
GTRecord sPkEnd = deserializeGTRecord(in, sInfo);
- List<GTRecord> sFuzzyKeys = Lists.newArrayList();
+ List<FuzzyKeyGTRecord> sFuzzyKeys = Lists.newArrayList();
int sFuzzyKeySize = BytesUtil.readVInt(in);
for (int i = 0; i < sFuzzyKeySize; i++) {
- sFuzzyKeys.add(deserializeGTRecord(in, sInfo));
+ sFuzzyKeys.add(deserializeFuzzyKeyGTRecord(in, sInfo));
}
GTScanRange sRange = new GTScanRange(sPkStart, sPkEnd, sFuzzyKeys);
sRanges.add(sRange);
@@ -311,7 +311,6 @@ public class GTScanRequest {
for (ByteArray col : gtRecord.cols) {
col.exportData(out);
}
- ImmutableBitSet.serializer.serialize(gtRecord.maskForEqualHashComp, out);
}
private GTRecord deserializeGTRecord(ByteBuffer in, GTInfo sInfo) {
@@ -320,8 +319,18 @@ public class GTScanRequest {
for (int i = 0; i < colLength; i++) {
sCols[i] = ByteArray.importData(in);
}
+ return new GTRecord(sInfo, sCols);
+ }
+
+ private void serializeFuzzyKeyGTRecord(FuzzyKeyGTRecord gtRecord, ByteBuffer out) {
+ serializeGTRecord(gtRecord,out);
+ ImmutableBitSet.serializer.serialize(gtRecord.maskForEqualHashComp, out);
+ }
+
+ private FuzzyKeyGTRecord deserializeFuzzyKeyGTRecord(ByteBuffer in, GTInfo sInfo) {
+ GTRecord temp = deserializeGTRecord(in,sInfo);
ImmutableBitSet sMaskForEqualHashComp = ImmutableBitSet.serializer.deserialize(in);
- return new GTRecord(sInfo, sMaskForEqualHashComp, sCols);
+ return new FuzzyKeyGTRecord(temp.info,temp.cols, sMaskForEqualHashComp);
}
};
http://git-wip-us.apache.org/repos/asf/kylin/blob/64b5f86f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index af5d4b7..015edc6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -40,6 +40,7 @@ import org.apache.kylin.cube.kv.RowKeyEncoder;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
import org.apache.kylin.cube.model.HBaseMappingDesc;
+import org.apache.kylin.gridtable.FuzzyKeyGTRecord;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRange;
@@ -92,7 +93,7 @@ public abstract class CubeHBaseRPC implements IGTStorage {
return scan;
}
- private RawScan preparedHBaseScan(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> fuzzyKeys, ImmutableBitSet selectedColBlocks) {
+ private RawScan preparedHBaseScan(GTRecord pkStart, GTRecord pkEnd, List<FuzzyKeyGTRecord> fuzzyKeys, ImmutableBitSet selectedColBlocks) {
final List<Pair<byte[], byte[]>> selectedColumns = makeHBaseColumns(selectedColBlocks);
LazyRowKeyEncoder encoder = new LazyRowKeyEncoder(cubeSeg, cuboid);
@@ -136,13 +137,13 @@ public abstract class CubeHBaseRPC implements IGTStorage {
* translate GTRecord format fuzzy keys to hbase expected format
* @return
*/
- private List<Pair<byte[], byte[]>> translateFuzzyKeys(List<GTRecord> fuzzyKeys) {
+ private List<Pair<byte[], byte[]>> translateFuzzyKeys(List<FuzzyKeyGTRecord> fuzzyKeys) {
if (fuzzyKeys == null || fuzzyKeys.isEmpty()) {
return Collections.emptyList();
}
List<Pair<byte[], byte[]>> ret = Lists.newArrayList();
- for (GTRecord gtRecordFuzzyKey : fuzzyKeys) {
+ for (FuzzyKeyGTRecord gtRecordFuzzyKey : fuzzyKeys) {
byte[] hbaseFuzzyKey = fuzzyKeyEncoder.createBuf();
byte[] hbaseFuzzyMask = fuzzyMaskEncoder.createBuf();