You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/06/01 10:19:25 UTC
incubator-kylin git commit: KYLIN-804, Retire BitSet in GridTable
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8.0 46d21544f -> 7f5290b36
KYLIN-804, Retire BitSet in GridTable
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7f5290b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7f5290b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7f5290b3
Branch: refs/heads/0.8.0
Commit: 7f5290b36abfe8942fcc1ff68f002d8061594f2b
Parents: 46d2154
Author: Yang Li <li...@apache.org>
Authored: Mon Jun 1 08:06:57 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Mon Jun 1 16:18:31 2015 +0800
----------------------------------------------------------------------
.../kylin/common/util/ImmutableBitSet.java | 120 +++++++++++++++++++
.../kylin/job/inmemcubing/InMemCubeBuilder.java | 39 +++---
.../kylin/job/streaming/CubeStreamBuilder.java | 4 +-
.../job/inmemcubing/InMemCubeBuilderTest.java | 9 +-
.../storage/cube/CubeHBaseReadonlyStore.java | 16 +--
.../apache/kylin/storage/cube/CubeScanner.java | 15 +--
.../storage/cube/CuboidToGridTableMapping.java | 18 +--
.../storage/gridtable/GTAggregateScanner.java | 67 ++++++-----
.../kylin/storage/gridtable/GTComboStore.java | 4 +-
.../apache/kylin/storage/gridtable/GTInfo.java | 63 +++++-----
.../storage/gridtable/GTInvertedIndex.java | 32 ++---
.../kylin/storage/gridtable/GTRawScanner.java | 4 +-
.../kylin/storage/gridtable/GTRecord.java | 102 +++++++++-------
.../kylin/storage/gridtable/GTRowBlock.java | 11 +-
.../kylin/storage/gridtable/GTScanRange.java | 7 +-
.../storage/gridtable/GTScanRangePlanner.java | 12 +-
.../kylin/storage/gridtable/GTScanRequest.java | 37 +++---
.../kylin/storage/gridtable/IGTStore.java | 5 +-
.../gridtable/diskstore/GTDiskStore.java | 24 ++--
.../gridtable/memstore/GTMemDiskStore.java | 12 +-
.../gridtable/memstore/GTSimpleMemStore.java | 4 +-
.../storage/gridtable/DictGridTableTest.java | 7 +-
.../storage/gridtable/SimpleGridTableTest.java | 7 +-
23 files changed, 392 insertions(+), 227 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java b/common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
new file mode 100644
index 0000000..807a03d
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
@@ -0,0 +1,120 @@
+package org.apache.kylin.common.util;
+
+import java.util.BitSet;
+
+public class ImmutableBitSet {
+
+ public static final ImmutableBitSet EMPTY = new ImmutableBitSet(new BitSet());
+
+ final private BitSet set;
+ final private int[] arr;
+
+ public ImmutableBitSet(int index) {
+ this(newBitSet(index));
+ }
+
+ private static BitSet newBitSet(int index) {
+ BitSet set = new BitSet(index);
+ set.set(index);
+ return set;
+ }
+
+ public ImmutableBitSet(int indexFrom, int indexTo) {
+ this(newBitSet(indexFrom, indexTo));
+ }
+
+ private static BitSet newBitSet(int indexFrom, int indexTo) {
+ BitSet set = new BitSet(indexTo);
+ set.set(indexFrom, indexTo);
+ return set;
+ }
+
+ public ImmutableBitSet(BitSet set) {
+ this.set = (BitSet) set.clone();
+ this.arr = new int[set.cardinality()];
+
+ int j = 0;
+ for (int i = set.nextSetBit(0); i >= 0; i = set.nextSetBit(i + 1)) {
+ arr[j++] = i;
+ }
+ }
+
+ public int trueBitCount() {
+ return arr.length;
+ }
+
+ public int trueBitAt(int i) {
+ return arr[i];
+ }
+
+ public BitSet mutable() {
+ return (BitSet) set.clone();
+ }
+
+ public ImmutableBitSet set(int bitIndex) {
+ return set(bitIndex, true);
+ }
+
+ public ImmutableBitSet set(int bitIndex, boolean value) {
+ if (set.get(bitIndex) == value) {
+ return this;
+ } else {
+ BitSet mutable = mutable();
+ mutable.set(bitIndex, value);
+ return new ImmutableBitSet(mutable);
+ }
+ }
+
+ public ImmutableBitSet or(ImmutableBitSet another) {
+ BitSet mutable = mutable();
+ mutable.or(another.set);
+ return new ImmutableBitSet(mutable);
+ }
+
+ public ImmutableBitSet andNot(ImmutableBitSet another) {
+ BitSet mutable = mutable();
+ mutable.andNot(another.set);
+ return new ImmutableBitSet(mutable);
+ }
+
+ @Override
+ public int hashCode() {
+ return set.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+
+ ImmutableBitSet other = (ImmutableBitSet) obj;
+ return this.set.equals(other.set);
+ }
+
+ @Override
+ public String toString() {
+ return set.toString();
+ }
+
+ // ============================================================================
+
+ public boolean get(int bitIndex) {
+ return set.get(bitIndex);
+ }
+
+ public int cardinality() {
+ return set.cardinality();
+ }
+
+ public boolean intersects(ImmutableBitSet another) {
+ return set.intersects(another.set);
+ }
+
+ public boolean isEmpty() {
+ return set.isEmpty();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/job/src/main/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilder.java b/job/src/main/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilder.java
index 38318e2..567716a 100644
--- a/job/src/main/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilder.java
@@ -34,6 +34,7 @@ import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
@@ -157,19 +158,20 @@ public class InMemCubeBuilder implements Runnable {
private GridTable newGridTableByCuboidID(long cuboidID) throws IOException {
GTInfo info = CubeGridTable.newGTInfo(cubeDesc, cuboidID, dictionaryMap);
- // could use a real budget controller, but experiment shows write directly to disk is a few ms faster
+ // could use a real budget controller, but experiment shows write directly to disk is the same fast
+ // GTMemDiskStore store = new GTMemDiskStore(info, memBudget == null ? MemoryBudgetController.ZERO_BUDGET : memBudget);
GTMemDiskStore store = new GTMemDiskStore(info, MemoryBudgetController.ZERO_BUDGET);
GridTable gridTable = new GridTable(info, store);
return gridTable;
}
- private Pair<BitSet, BitSet> getDimensionAndMetricColumnBitSet(long cuboidId) {
+ private Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet(long cuboidId) {
BitSet bitSet = BitSet.valueOf(new long[] { cuboidId });
BitSet dimension = new BitSet();
dimension.set(0, bitSet.cardinality());
BitSet metrics = new BitSet();
metrics.set(bitSet.cardinality(), bitSet.cardinality() + this.measureCount);
- return new Pair<BitSet, BitSet>(dimension, metrics);
+ return new Pair<ImmutableBitSet, ImmutableBitSet>(new ImmutableBitSet(dimension), new ImmutableBitSet(metrics));
}
@Override
@@ -396,7 +398,7 @@ public class InMemCubeBuilder implements Runnable {
int mbBefore = getSystemAvailMB();
int mbAfter = 0;
- Pair<BitSet, BitSet> dimensionMetricsBitSet = getDimensionAndMetricColumnBitSet(baseCuboidId);
+ Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = getDimensionAndMetricColumnBitSet(baseCuboidId);
GTScanRequest req = new GTScanRequest(baseCuboid.getInfo(), null, dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), metricsAggrFuncs, null);
GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req);
@@ -465,10 +467,10 @@ public class InMemCubeBuilder implements Runnable {
}
private CuboidResult aggregateCuboid(CuboidResult parent, long cuboidId) throws IOException {
- Pair<BitSet, BitSet> columnBitSets = getDimensionAndMetricColumnBitSet(parent.cuboidId);
- BitSet parentDimensions = columnBitSets.getFirst();
- BitSet measureColumns = columnBitSets.getSecond();
- BitSet childDimensions = (BitSet) parentDimensions.clone();
+ Pair<ImmutableBitSet, ImmutableBitSet> columnBitSets = getDimensionAndMetricColumnBitSet(parent.cuboidId);
+ ImmutableBitSet parentDimensions = columnBitSets.getFirst();
+ ImmutableBitSet measureColumns = columnBitSets.getSecond();
+ ImmutableBitSet childDimensions = parentDimensions;
long mask = Long.highestOneBit(parent.cuboidId);
long childCuboidId = cuboidId;
@@ -478,7 +480,7 @@ public class InMemCubeBuilder implements Runnable {
if ((mask & parent.cuboidId) > 0) {
if ((mask & childCuboidId) == 0) {
// this dim will be aggregated
- childDimensions.set(index, false);
+ childDimensions = childDimensions.set(index, false);
}
index++;
}
@@ -488,7 +490,7 @@ public class InMemCubeBuilder implements Runnable {
return scanAndAggregateGridTable(parent.table, cuboidId, childDimensions, measureColumns);
}
- private CuboidResult scanAndAggregateGridTable(GridTable gridTable, long cuboidId, BitSet aggregationColumns, BitSet measureColumns) throws IOException {
+ private CuboidResult scanAndAggregateGridTable(GridTable gridTable, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException {
long startTime = System.currentTimeMillis();
logger.info("Calculating cuboid " + cuboidId);
@@ -497,26 +499,26 @@ public class InMemCubeBuilder implements Runnable {
GridTable newGridTable = newGridTableByCuboidID(cuboidId);
GTBuilder builder = newGridTable.rebuild();
- BitSet allNeededColumns = new BitSet();
- allNeededColumns.or(aggregationColumns);
- allNeededColumns.or(measureColumns);
+ ImmutableBitSet allNeededColumns = aggregationColumns.or(measureColumns);
GTRecord newRecord = new GTRecord(newGridTable.getInfo());
int count = 0;
ByteArray byteArray = new ByteArray(8);
ByteBuffer byteBuffer = ByteBuffer.allocate(8);
try {
- BitSet dependentMetrics = new BitSet(allNeededColumns.cardinality());
+ BitSet dependentMetricsBS = new BitSet(allNeededColumns.cardinality());
for (Integer i : dependentMeasures.keySet()) {
- dependentMetrics.set((allNeededColumns.cardinality() - measureCount + dependentMeasures.get(i)));
+ dependentMetricsBS.set((allNeededColumns.cardinality() - measureCount + dependentMeasures.get(i)));
}
+ ImmutableBitSet dependentMetrics = new ImmutableBitSet(dependentMetricsBS);
Object[] hllObjects = new Object[dependentMeasures.keySet().size()];
for (GTRecord record : scanner) {
count++;
- for (int i = allNeededColumns.nextSetBit(0), index = 0; i >= 0; i = allNeededColumns.nextSetBit(i + 1), index++) {
- newRecord.set(index, record.get(i));
+ for (int i = 0; i < allNeededColumns.trueBitCount(); i++) {
+ int c = allNeededColumns.trueBitAt(i);
+ newRecord.set(i, record.get(c));
}
if (dependentMeasures.size() > 0) {
@@ -524,7 +526,8 @@ public class InMemCubeBuilder implements Runnable {
newRecord.getValues(dependentMetrics, hllObjects);
for (Integer i : dependentMeasures.keySet()) {
- for (int index = 0, c = dependentMetrics.nextSetBit(0); c >= 0; index++, c = dependentMetrics.nextSetBit(c + 1)) {
+ for (int index = 0; index < dependentMetrics.trueBitCount(); index++) {
+ int c = dependentMetrics.trueBitAt(index);
if (c == allNeededColumns.cardinality() - measureCount + dependentMeasures.get(i)) {
assert hllObjects[index] instanceof HyperLogLogPlusCounter; // currently only HLL is allowed
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
index 07e19b8..a164396 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
@@ -31,6 +31,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -196,8 +197,7 @@ public class CubeStreamBuilder extends StreamBuilder {
public void write(long cuboidId, GTRecord record) throws IOException {
final ByteBuffer key = createKey(cuboidId, record);
final CuboidToGridTableMapping mapping = new CuboidToGridTableMapping(Cuboid.findById(cubeDesc, cuboidId));
- final BitSet bitSet = new BitSet();
- bitSet.set(mapping.getDimensionCount(), mapping.getColumnCount());
+ final ImmutableBitSet bitSet = new ImmutableBitSet(mapping.getDimensionCount(), mapping.getColumnCount());
for (int i = 0; i < nColumns; i++) {
final KeyValue keyValue = keyValueCreators.get(i).create(key.array(), 0, key.position(), record.getValues(bitSet, new Object[bitSet.cardinality()]));
final Put put = new Put(copy(key.array(), 0, key.position()));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/job/src/test/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilderTest.java b/job/src/test/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilderTest.java
index 4e94ffa..96927a7 100644
--- a/job/src/test/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilderTest.java
+++ b/job/src/test/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilderTest.java
@@ -78,6 +78,9 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
final int inputRows = 70000;
final int threads = 4;
+ System.out.println("Hit enter to start");
+ System.in.read();
+
final CubeInstance cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty");
final String flatTable = "../examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv";
@@ -107,7 +110,7 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
Set<String>[] distinctSets = new Set[nColumns];
for (int i = 0; i < nColumns; i++)
distinctSets[i] = new TreeSet<String>();
-
+
// get distinct values on each column
FileTableReader reader = new FileTableReader(flatTable, nColumns);
while (count > 0 && reader.next()) {
@@ -116,12 +119,12 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
distinctSets[i].add(row[i]);
}
reader.close();
-
+
List<String[]> distincts = new ArrayList<String[]>();
for (int i = 0; i < nColumns; i++) {
distincts.add((String[]) distinctSets[i].toArray(new String[distinctSets[i].size()]));
}
-
+
// output with random data
Random rand = new Random();
for (; count > 0; count--) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
index cd0904b..261e85d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
@@ -2,7 +2,6 @@ package org.apache.kylin.storage.cube;
import java.io.IOException;
import java.util.Arrays;
-import java.util.BitSet;
import java.util.Iterator;
import java.util.List;
@@ -12,10 +11,11 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowConstants;
@@ -65,12 +65,11 @@ public class CubeHBaseReadonlyStore implements IGTStore {
}
@Override
- public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColumnBlocks, GTScanRequest additionalPushDown) throws IOException {
+ public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, ImmutableBitSet selectedColumnBlocks, GTScanRequest additionalPushDown) throws IOException {
// TODO enable coprocessor
// primary key (also the 0th column block) is always selected
- final BitSet selectedColBlocks = (BitSet) selectedColumnBlocks.clone();
- selectedColBlocks.set(0);
+ final ImmutableBitSet selectedColBlocks = selectedColumnBlocks.set(0);
// globally shared connection, does not require close
HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
@@ -102,7 +101,8 @@ public class CubeHBaseReadonlyStore implements IGTStore {
// metrics
int hbaseColIdx = 0;
- for (int colBlockIdx = selectedColBlocks.nextSetBit(1); colBlockIdx >= 0; colBlockIdx = selectedColBlocks.nextSetBit(colBlockIdx + 1)) {
+ for (int i = 0; i < selectedColBlocks.trueBitCount(); i++) {
+ int colBlockIdx = selectedColBlocks.trueBitAt(i);
Pair<byte[], byte[]> hbaseColumn = hbaseColumns.get(hbaseColIdx++);
Cell cell = result.getColumnLatestCell(hbaseColumn.getFirst(), hbaseColumn.getSecond());
oneBlock.getCellBlock(colBlockIdx).set(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
@@ -155,7 +155,7 @@ public class CubeHBaseReadonlyStore implements IGTStore {
return buf;
}
- private List<Pair<byte[], byte[]>> makeHBaseColumns(BitSet selectedColBlocks) {
+ private List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) {
List<Pair<byte[], byte[]>> result = Lists.newArrayList();
int colBlockIdx = 1; // start from 1; the 0th column block is primary key which maps to rowkey
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/cube/CubeScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeScanner.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeScanner.java
index 9751135..038df9d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeScanner.java
@@ -8,6 +8,7 @@ import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -42,9 +43,9 @@ public class CubeScanner implements IGTScanner {
CuboidToGridTableMapping mapping = new CuboidToGridTableMapping(cuboid);
TupleFilter gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, mapping.getCuboidDimensionsInGTOrder(), groups);
- BitSet gtDimensions = makeGridTableColumns(mapping, dimensions);
- BitSet gtAggrGroups = makeGridTableColumns(mapping, groups);
- BitSet gtAggrMetrics = makeGridTableColumns(mapping, metrics);
+ ImmutableBitSet gtDimensions = makeGridTableColumns(mapping, dimensions);
+ ImmutableBitSet gtAggrGroups = makeGridTableColumns(mapping, groups);
+ ImmutableBitSet gtAggrMetrics = makeGridTableColumns(mapping, metrics);
String[] gtAggrFuncs = makeAggrFuncs(metrics);
GTScanRangePlanner scanRangePlanner = new GTScanRangePlanner(info);
@@ -58,17 +59,17 @@ public class CubeScanner implements IGTScanner {
scanner = new Scanner();
}
- private BitSet makeGridTableColumns(CuboidToGridTableMapping mapping, Set<TblColRef> dimensions) {
+ private ImmutableBitSet makeGridTableColumns(CuboidToGridTableMapping mapping, Set<TblColRef> dimensions) {
BitSet result = new BitSet();
for (TblColRef dim : dimensions) {
int idx = mapping.getIndexOf(dim);
if (idx >= 0)
result.set(idx);
}
- return result;
+ return new ImmutableBitSet(result);
}
- private BitSet makeGridTableColumns(CuboidToGridTableMapping mapping, Collection<FunctionDesc> metrics) {
+ private ImmutableBitSet makeGridTableColumns(CuboidToGridTableMapping mapping, Collection<FunctionDesc> metrics) {
BitSet result = new BitSet();
for (FunctionDesc metric : metrics) {
int idx = mapping.getIndexOf(metric);
@@ -76,7 +77,7 @@ public class CubeScanner implements IGTScanner {
throw new IllegalStateException(metric + " not found in " + mapping);
result.set(idx);
}
- return result;
+ return new ImmutableBitSet(result);
}
private String[] makeAggrFuncs(Collection<FunctionDesc> metrics) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/cube/CuboidToGridTableMapping.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CuboidToGridTableMapping.java b/storage/src/main/java/org/apache/kylin/storage/cube/CuboidToGridTableMapping.java
index 51079c4..4388eac 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CuboidToGridTableMapping.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CuboidToGridTableMapping.java
@@ -4,6 +4,7 @@ import java.util.BitSet;
import java.util.List;
import java.util.Map;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
@@ -22,11 +23,11 @@ public class CuboidToGridTableMapping {
final private Cuboid cuboid;
private List<DataType> gtDataTypes;
- private List<BitSet> gtColBlocks;
+ private List<ImmutableBitSet> gtColBlocks;
private int nDimensions;
private Map<TblColRef, Integer> dim2gt;
- private BitSet gtPrimaryKey;
+ private ImmutableBitSet gtPrimaryKey;
private int nMetrics;
private ListMultimap<FunctionDesc, Integer> metrics2gt; // because count distinct may have a holistic version
@@ -43,13 +44,14 @@ public class CuboidToGridTableMapping {
// dimensions
dim2gt = Maps.newHashMap();
- gtPrimaryKey = new BitSet();
+ BitSet pk = new BitSet();
for (TblColRef dimension : cuboid.getColumns()) {
gtDataTypes.add(dimension.getType());
dim2gt.put(dimension, gtColIdx);
- gtPrimaryKey.set(gtColIdx);
+ pk.set(gtColIdx);
gtColIdx++;
}
+ gtPrimaryKey = new ImmutableBitSet(pk);
gtColBlocks.add(gtPrimaryKey);
nDimensions = gtColIdx;
@@ -73,7 +75,7 @@ public class CuboidToGridTableMapping {
colBlock.set(gtColIdx);
gtColIdx++;
}
- gtColBlocks.add(colBlock);
+ gtColBlocks.add(new ImmutableBitSet(colBlock));
}
}
nMetrics = gtColIdx - nDimensions;
@@ -96,12 +98,12 @@ public class CuboidToGridTableMapping {
return (DataType[]) gtDataTypes.toArray(new DataType[gtDataTypes.size()]);
}
- public BitSet getPrimaryKey() {
+ public ImmutableBitSet getPrimaryKey() {
return gtPrimaryKey;
}
- public BitSet[] getColumnBlocks() {
- return (BitSet[]) gtColBlocks.toArray(new BitSet[gtColBlocks.size()]);
+ public ImmutableBitSet[] getColumnBlocks() {
+ return (ImmutableBitSet[]) gtColBlocks.toArray(new ImmutableBitSet[gtColBlocks.size()]);
}
public int getIndexOf(TblColRef dimension) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
index b08a894..c34adc7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
@@ -1,21 +1,20 @@
package org.apache.kylin.storage.gridtable;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.BitSet;
import java.util.Comparator;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.SortedMap;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Maps;
+
@SuppressWarnings({ "rawtypes", "unchecked" })
public class GTAggregateScanner implements IGTScanner {
@@ -23,9 +22,9 @@ public class GTAggregateScanner implements IGTScanner {
private static final Logger logger = LoggerFactory.getLogger(GTAggregateScanner.class);
final GTInfo info;
- final BitSet dimensions; // dimensions to return, can be more than group by
- final BitSet groupBy;
- final BitSet metrics;
+ final ImmutableBitSet dimensions; // dimensions to return, can be more than group by
+ final ImmutableBitSet groupBy;
+ final ImmutableBitSet metrics;
final String[] metricsAggrFuncs;
final IGTScanner inputScanner;
@@ -36,8 +35,7 @@ public class GTAggregateScanner implements IGTScanner {
throw new IllegalStateException();
this.info = inputScanner.getInfo();
- this.dimensions = (BitSet) req.getColumns().clone();
- this.dimensions.andNot(req.getAggrMetrics());
+ this.dimensions = req.getColumns().andNot(req.getAggrMetrics());
this.groupBy = req.getAggrGroupBy();
this.metrics = req.getAggrMetrics();
this.metricsAggrFuncs = req.getAggrMetricsFuncs();
@@ -93,7 +91,8 @@ public class GTAggregateScanner implements IGTScanner {
@Override
public int compare(byte[] o1, byte[] o2) {
int result = 0;
- Preconditions.checkArgument(keyLength == o1.length && keyLength == o2.length);
+ // profiler shows this check is slow
+ // Preconditions.checkArgument(keyLength == o1.length && keyLength == o2.length);
for (int i = 0; i < keyLength; ++i) {
if (compareMask[i]) {
int a = (o1[i] & 0xff);
@@ -113,16 +112,18 @@ public class GTAggregateScanner implements IGTScanner {
private boolean[] createCompareMask() {
int keyLength = 0;
- for (int i = dimensions.nextSetBit(0); i >= 0; i = dimensions.nextSetBit(i + 1)) {
- int l = info.codeSystem.maxCodeLength(i);
+ for (int i = 0; i < dimensions.trueBitCount(); i++) {
+ int c = dimensions.trueBitAt(i);
+ int l = info.codeSystem.maxCodeLength(c);
keyLength += l;
}
boolean[] mask = new boolean[keyLength];
int p = 0;
- for (int i = dimensions.nextSetBit(0); i >= 0; i = dimensions.nextSetBit(i + 1)) {
- int l = info.codeSystem.maxCodeLength(i);
- boolean m = groupBy.get(i) ? true : false;
+ for (int i = 0; i < dimensions.trueBitCount(); i++) {
+ int c = dimensions.trueBitAt(i);
+ int l = info.codeSystem.maxCodeLength(c);
+ boolean m = groupBy.get(c) ? true : false;
for (int j = 0; j < l; j++) {
mask[p++] = m;
}
@@ -133,9 +134,10 @@ public class GTAggregateScanner implements IGTScanner {
private byte[] createKey(GTRecord record) {
byte[] result = new byte[keyLength];
int offset = 0;
- for (int i = dimensions.nextSetBit(0); i >= 0; i = dimensions.nextSetBit(i + 1)) {
- final ByteArray byteArray = record.cols[i];
- final int columnLength = info.codeSystem.maxCodeLength(i);
+ for (int i = 0; i < dimensions.trueBitCount(); i++) {
+ int c = dimensions.trueBitAt(i);
+ final ByteArray byteArray = record.cols[c];
+ final int columnLength = info.codeSystem.maxCodeLength(c);
System.arraycopy(byteArray.array(), byteArray.offset(), result, offset, byteArray.length());
offset += columnLength;
}
@@ -148,14 +150,14 @@ public class GTAggregateScanner implements IGTScanner {
MeasureAggregator[] aggrs = aggBufMap.get(key);
if (aggrs == null) {
aggrs = new MeasureAggregator[metricsAggrFuncs.length];
- for (int i = 0, col = -1; i < aggrs.length; i++) {
- col = metrics.nextSetBit(col + 1);
+ for (int i = 0; i < aggrs.length; i++) {
+ int col = metrics.trueBitAt(i);
aggrs[i] = info.codeSystem.newMetricsAggregator(metricsAggrFuncs[i], col);
}
aggBufMap.put(key, aggrs);
}
- for (int i = 0, col = -1; i < aggrs.length; i++) {
- col = metrics.nextSetBit(col + 1);
+ for (int i = 0; i < aggrs.length; i++) {
+ int col = metrics.trueBitAt(i);
Object metrics = info.codeSystem.decodeColumnValue(col, r.cols[col].asBuffer());
aggrs[i].aggregate(metrics);
}
@@ -192,14 +194,15 @@ public class GTAggregateScanner implements IGTScanner {
private void create(byte[] key, MeasureAggregator[] value) {
int offset = 0;
- for (int i = dimensions.nextSetBit(0); i >= 0; i = dimensions.nextSetBit(i + 1)) {
- final int columnLength = info.codeSystem.maxCodeLength(i);
- secondRecord.set(i, new ByteArray(key, offset, columnLength));
+ for (int i = 0; i < dimensions.trueBitCount(); i++) {
+ int c = dimensions.trueBitAt(i);
+ final int columnLength = info.codeSystem.maxCodeLength(c);
+ secondRecord.set(c, new ByteArray(key, offset, columnLength));
offset += columnLength;
}
metricsBuf.clear();
- for (int i = 0, col = -1; i < value.length; i++) {
- col = metrics.nextSetBit(col + 1);
+ for (int i = 0; i < value.length; i++) {
+ int col = metrics.trueBitAt(i);
int pos = metricsBuf.position();
info.codeSystem.encodeColumnValue(col, value[i].getState(), metricsBuf);
secondRecord.cols[col].set(metricsBuf.array(), pos, metricsBuf.position() - pos);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
index 92707ed..42b577a 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
@@ -1,8 +1,8 @@
package org.apache.kylin.storage.gridtable;
import java.io.IOException;
-import java.util.BitSet;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.storage.gridtable.diskstore.GTDiskStore;
import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore;
import org.slf4j.Logger;
@@ -100,7 +100,7 @@ public class GTComboStore implements IGTStore {
}
@Override
- public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException {
+ public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, ImmutableBitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException {
return getCurrent().scan(pkStart, pkEnd, selectedColBlocks, additionalPushDown);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
index 5892296..f552e19 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
@@ -5,6 +5,7 @@ import java.util.BitSet;
import java.util.Iterator;
import java.util.LinkedList;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
@@ -20,14 +21,14 @@ public class GTInfo {
// column schema
int nColumns;
DataType[] colTypes;
- BitSet colAll;
- BitSet colPreferIndex;
+ ImmutableBitSet colAll;
+ ImmutableBitSet colPreferIndex;
transient TblColRef[] colRefs;
// grid info
- BitSet primaryKey; // order by, uniqueness is not required
- BitSet[] colBlocks; // primary key must be the first column block
- BitSet colBlocksAll;
+ ImmutableBitSet primaryKey; // order by, uniqueness is not required
+ ImmutableBitSet[] colBlocks; // primary key must be the first column block
+ ImmutableBitSet colBlocksAll;
int rowBlockSize; // 0: disable row block
// sharding & rowkey
@@ -53,7 +54,7 @@ public class GTInfo {
return colTypes[i];
}
- public BitSet getPrimaryKey() {
+ public ImmutableBitSet getPrimaryKey() {
return primaryKey;
}
@@ -73,10 +74,11 @@ public class GTInfo {
return getMaxColumnLength(colAll);
}
- public int getMaxColumnLength(BitSet selectedCols) {
+ public int getMaxColumnLength(ImmutableBitSet selectedCols) {
int result = 0;
- for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
- result += codeSystem.maxCodeLength(i);
+ for (int i = 0; i < selectedCols.trueBitCount(); i++) {
+ int c = selectedCols.trueBitAt(i);
+ result += codeSystem.maxCodeLength(c);
}
return result;
}
@@ -88,18 +90,18 @@ public class GTInfo {
return max;
}
- public BitSet selectColumnBlocks(BitSet columns) {
+ public ImmutableBitSet selectColumnBlocks(ImmutableBitSet columns) {
if (columns == null)
columns = colAll;
BitSet result = new BitSet();
for (int i = 0; i < colBlocks.length; i++) {
- BitSet cb = colBlocks[i];
+ ImmutableBitSet cb = colBlocks[i];
if (cb.intersects(columns)) {
result.set(i);
}
}
- return result;
+ return new ImmutableBitSet(result);
}
public TblColRef colRef(int i) {
@@ -132,21 +134,18 @@ public class GTInfo {
}
private void validateColumnBlocks() {
- colAll = new BitSet();
- colAll.flip(0, nColumns);
+ colAll = new ImmutableBitSet(0, nColumns);
if (colBlocks == null) {
- colBlocks = new BitSet[2];
+ colBlocks = new ImmutableBitSet[2];
colBlocks[0] = primaryKey;
- colBlocks[1] = (BitSet) colAll.clone();
- colBlocks[1].andNot(primaryKey);
+ colBlocks[1] = colAll.andNot(primaryKey);
}
- colBlocksAll = new BitSet();
- colBlocksAll.flip(0, colBlocks.length);
+ colBlocksAll = new ImmutableBitSet(0, colBlocks.length);
if (colPreferIndex == null)
- colPreferIndex = new BitSet();
+ colPreferIndex = ImmutableBitSet.EMPTY;
// column blocks must not overlap
for (int i = 0; i < colBlocks.length; i++) {
@@ -157,9 +156,9 @@ public class GTInfo {
}
// column block must cover all columns
- BitSet merge = new BitSet();
+ ImmutableBitSet merge = ImmutableBitSet.EMPTY;
for (int i = 0; i < colBlocks.length; i++) {
- merge.or(colBlocks[i]);
+ merge = merge.or(colBlocks[i]);
}
if (merge.equals(colAll) == false)
throw new IllegalStateException();
@@ -169,14 +168,14 @@ public class GTInfo {
throw new IllegalStateException();
// drop empty column block
- LinkedList<BitSet> tmp = new LinkedList<BitSet>(Arrays.asList(colBlocks));
- Iterator<BitSet> it = tmp.iterator();
+ LinkedList<ImmutableBitSet> list = new LinkedList<ImmutableBitSet>(Arrays.asList(colBlocks));
+ Iterator<ImmutableBitSet> it = list.iterator();
while (it.hasNext()) {
- BitSet cb = it.next();
+ ImmutableBitSet cb = it.next();
if (cb.isEmpty())
it.remove();
}
- colBlocks = (BitSet[]) tmp.toArray(new BitSet[tmp.size()]);
+ colBlocks = (ImmutableBitSet[]) list.toArray(new ImmutableBitSet[list.size()]);
}
public static class Builder {
@@ -206,16 +205,16 @@ public class GTInfo {
}
/** required */
- public Builder setPrimaryKey(BitSet primaryKey) {
- info.primaryKey = (BitSet) primaryKey.clone();
+ public Builder setPrimaryKey(ImmutableBitSet primaryKey) {
+ info.primaryKey = primaryKey;
return this;
}
/** optional */
- public Builder enableColumnBlock(BitSet[] columnBlocks) {
- info.colBlocks = new BitSet[columnBlocks.length];
+ public Builder enableColumnBlock(ImmutableBitSet[] columnBlocks) {
+ info.colBlocks = new ImmutableBitSet[columnBlocks.length];
for (int i = 0; i < columnBlocks.length; i++) {
- info.colBlocks[i] = (BitSet) columnBlocks[i].clone();
+ info.colBlocks[i] = columnBlocks[i];
}
return this;
}
@@ -233,7 +232,7 @@ public class GTInfo {
}
/** optional */
- public Builder setColumnPreferIndex(BitSet colPreferIndex) {
+ public Builder setColumnPreferIndex(ImmutableBitSet colPreferIndex) {
info.colPreferIndex = colPreferIndex;
return this;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
index 2756659..4196528 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInvertedIndex.java
@@ -2,12 +2,12 @@ package org.apache.kylin.storage.gridtable;
import it.uniroma3.mat.extendedset.intset.ConciseSet;
-import java.util.BitSet;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -23,8 +23,8 @@ import org.apache.kylin.metadata.filter.TupleFilter;
public class GTInvertedIndex {
private final GTInfo info;
- private final BitSet colPreferIndex;
- private final BitSet colBlocks;
+ private final ImmutableBitSet colPreferIndex;
+ private final ImmutableBitSet colBlocks;
private final GTInvertedIndexOfColumn[] index; // for each column
private volatile int nIndexedBlocks;
@@ -35,8 +35,9 @@ public class GTInvertedIndex {
this.colBlocks = info.selectColumnBlocks(colPreferIndex);
index = new GTInvertedIndexOfColumn[info.getColumnCount()];
- for (int i = colPreferIndex.nextSetBit(0); i >= 0; i = colPreferIndex.nextSetBit(i + 1)) {
- index[i] = new GTInvertedIndexOfColumn(info.codeSystem.getFilterCodeSystem());
+ for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
+ int c = colPreferIndex.trueBitAt(i);
+ index[c] = new GTInvertedIndexOfColumn(info.codeSystem.getFilterCodeSystem());
}
}
@@ -44,21 +45,24 @@ public class GTInvertedIndex {
@SuppressWarnings("unchecked")
Set<ByteArray>[] distinctValues = new Set[info.getColumnCount()];
- for (int i = colPreferIndex.nextSetBit(0); i >= 0; i = colPreferIndex.nextSetBit(i + 1)) {
- distinctValues[i] = new HashSet<ByteArray>();
+ for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
+ int c = colPreferIndex.trueBitAt(i);
+ distinctValues[c] = new HashSet<ByteArray>();
}
GTRowBlock.Reader reader = block.getReader(colBlocks);
GTRecord record = new GTRecord(info);
while (reader.hasNext()) {
reader.fetchNext(record);
- for (int i = colPreferIndex.nextSetBit(0); i >= 0; i = colPreferIndex.nextSetBit(i + 1)) {
- distinctValues[i].add(record.get(i));
+ for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
+ int c = colPreferIndex.trueBitAt(i);
+ distinctValues[c].add(record.get(c));
}
}
- for (int i = colPreferIndex.nextSetBit(0); i >= 0; i = colPreferIndex.nextSetBit(i + 1)) {
- index[i].add(distinctValues[i], block.getSequenceId());
+ for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
+ int c = colPreferIndex.trueBitAt(i);
+ index[c].add(distinctValues[c], block.getSequenceId());
}
nIndexedBlocks = Math.max(nIndexedBlocks, block.seqId + 1);
@@ -67,7 +71,7 @@ public class GTInvertedIndex {
public ConciseSet filter(TupleFilter filter) {
return filter(filter, nIndexedBlocks);
}
-
+
public ConciseSet filter(TupleFilter filter, int totalBlocks) {
// number of indexed blocks may increase as we do evaluation
int indexedBlocks = nIndexedBlocks;
@@ -110,7 +114,7 @@ public class GTInvertedIndex {
int col = col(filter);
if (index[col] == null)
return all();
-
+
switch (filter.getOperator()) {
case ISNULL:
return index[col].getNull();
@@ -185,7 +189,7 @@ public class GTInvertedIndex {
private ConciseSet all() {
return not(new ConciseSet());
}
-
+
private ConciseSet not(ConciseSet set) {
set.add(indexedBlocks);
set.complement();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
index 1e934b5..0313026 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
@@ -1,17 +1,17 @@
package org.apache.kylin.storage.gridtable;
import java.io.IOException;
-import java.util.BitSet;
import java.util.Iterator;
import java.util.NoSuchElementException;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.storage.gridtable.IGTStore.IGTStoreScanner;
public class GTRawScanner implements IGTScanner {
final GTInfo info;
final IGTStoreScanner storeScanner;
- final BitSet selectedColBlocks;
+ final ImmutableBitSet selectedColBlocks;
private GTRowBlock.Reader curBlockReader;
private GTRecord next;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
index 0e5c1b7..23d65bc 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
@@ -1,6 +1,7 @@
package org.apache.kylin.storage.gridtable;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.filter.IFilterCodeSystem;
import java.nio.ByteBuffer;
@@ -12,9 +13,9 @@ public class GTRecord implements Comparable<GTRecord> {
final GTInfo info;
final ByteArray[] cols;
- private BitSet maskForEqualHashComp;
+ private ImmutableBitSet maskForEqualHashComp;
- public GTRecord(GTInfo info, BitSet maskForEqualHashComp) {
+ public GTRecord(GTInfo info, ImmutableBitSet maskForEqualHashComp) {
this.info = info;
this.cols = new ByteArray[info.getColumnCount()];
for (int i = 0; i < cols.length; i++) {
@@ -48,10 +49,13 @@ public class GTRecord implements Comparable<GTRecord> {
}
/** set record to the codes of specified values, reuse given space to hold the codes */
- public GTRecord setValues(BitSet selectedColumns, ByteArray space, Object... values) {
+ public GTRecord setValues(ImmutableBitSet selectedCols, ByteArray space, Object... values) {
+ assert selectedCols.cardinality() == values.length;
+
ByteBuffer buf = space.asBuffer();
int pos = buf.position();
- for (int i = 0, c = selectedColumns.nextSetBit(0); c >= 0; i++, c = selectedColumns.nextSetBit(c + 1)) {
+ for (int i = 0; i < selectedCols.trueBitCount(); i++) {
+ int c = selectedCols.trueBitAt(i);
info.codeSystem.encodeColumnValue(c, values[i], buf);
int newPos = buf.position();
cols[c].set(buf.array(), buf.arrayOffset() + pos, newPos - pos);
@@ -66,9 +70,11 @@ public class GTRecord implements Comparable<GTRecord> {
}
/** decode and return the values of this record */
- public Object[] getValues(BitSet selectedColumns, Object[] result) {
- assert selectedColumns.cardinality() <= result.length;
- for (int i = 0, c = selectedColumns.nextSetBit(0); c >= 0; i++, c = selectedColumns.nextSetBit(c + 1)) {
+ public Object[] getValues(ImmutableBitSet selectedCols, Object[] result) {
+ assert selectedCols.cardinality() == result.length;
+
+ for (int i = 0; i < selectedCols.trueBitCount(); i++) {
+ int c = selectedCols.trueBitAt(i);
if (cols[c] == null || cols[c].array() == null) {
result[i] = null;
} else {
@@ -95,30 +101,32 @@ public class GTRecord implements Comparable<GTRecord> {
return copy(info.colAll);
}
- public GTRecord copy(BitSet selectedCols) {
+ public GTRecord copy(ImmutableBitSet selectedCols) {
int len = 0;
- for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
- len += cols[i].length();
+ for (int i = 0; i < selectedCols.trueBitCount(); i++) {
+ int c = selectedCols.trueBitAt(i);
+ len += cols[c].length();
}
byte[] space = new byte[len];
GTRecord copy = new GTRecord(info, this.maskForEqualHashComp);
int pos = 0;
- for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
- System.arraycopy(cols[i].array(), cols[i].offset(), space, pos, cols[i].length());
- copy.cols[i].set(space, pos, cols[i].length());
- pos += cols[i].length();
+ for (int i = 0; i < selectedCols.trueBitCount(); i++) {
+ int c = selectedCols.trueBitAt(i);
+ System.arraycopy(cols[c].array(), cols[c].offset(), space, pos, cols[c].length());
+ copy.cols[c].set(space, pos, cols[c].length());
+ pos += cols[c].length();
}
return copy;
}
- public BitSet maskForEqualHashComp() {
+ public ImmutableBitSet maskForEqualHashComp() {
return maskForEqualHashComp;
}
- public void maskForEqualHashComp(BitSet set) {
+ public void maskForEqualHashComp(ImmutableBitSet set) {
this.maskForEqualHashComp = set;
}
@@ -136,8 +144,9 @@ public class GTRecord implements Comparable<GTRecord> {
return false;
if (this.maskForEqualHashComp != o.maskForEqualHashComp)
return false;
- for (int i = maskForEqualHashComp.nextSetBit(0); i >= 0; i = maskForEqualHashComp.nextSetBit(i + 1)) {
- if (this.cols[i].equals(o.cols[i]) == false) {
+ for (int i = 0; i < maskForEqualHashComp.trueBitCount(); i++) {
+ int c = maskForEqualHashComp.trueBitAt(i);
+ if (this.cols[c].equals(o.cols[c]) == false) {
return false;
}
}
@@ -147,8 +156,9 @@ public class GTRecord implements Comparable<GTRecord> {
@Override
public int hashCode() {
int hash = 1;
- for (int i = maskForEqualHashComp.nextSetBit(0); i >= 0; i = maskForEqualHashComp.nextSetBit(i + 1)) {
- hash = (31 * hash) + cols[i].hashCode();
+ for (int i = 0; i < maskForEqualHashComp.trueBitCount(); i++) {
+ int c = maskForEqualHashComp.trueBitAt(i);
+ hash = (31 * hash) + cols[c].hashCode();
}
return hash;
}
@@ -160,8 +170,9 @@ public class GTRecord implements Comparable<GTRecord> {
IFilterCodeSystem<ByteArray> cs = info.codeSystem.getFilterCodeSystem();
int comp = 0;
- for (int i = maskForEqualHashComp.nextSetBit(0); i >= 0; i = maskForEqualHashComp.nextSetBit(i + 1)) {
- comp = cs.compare(cols[i], o.cols[i]);
+ for (int i = 0; i < maskForEqualHashComp.trueBitCount(); i++) {
+ int c = maskForEqualHashComp.trueBitAt(i);
+ comp = cs.compare(cols[c], o.cols[c]);
if (comp != 0)
return comp;
}
@@ -173,7 +184,7 @@ public class GTRecord implements Comparable<GTRecord> {
return toString(maskForEqualHashComp);
}
- public String toString(BitSet selectedColumns) {
+ public String toString(ImmutableBitSet selectedColumns) {
Object[] values = new Object[selectedColumns.cardinality()];
getValues(selectedColumns, values);
return Arrays.toString(values);
@@ -181,10 +192,11 @@ public class GTRecord implements Comparable<GTRecord> {
// ============================================================================
- public ByteArray exportColumns(BitSet selectedCols) {
+ public ByteArray exportColumns(ImmutableBitSet selectedCols) {
int len = 0;
- for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
- len += cols[i].length();
+ for (int i = 0; i < selectedCols.trueBitCount(); i++) {
+ int c = selectedCols.trueBitAt(i);
+ len += cols[c].length();
}
ByteArray buf = ByteArray.allocate(len);
@@ -193,19 +205,21 @@ public class GTRecord implements Comparable<GTRecord> {
}
/** write data to given buffer, like serialize */
- public void exportColumns(BitSet selectedCols, ByteArray buf) {
+ public void exportColumns(ImmutableBitSet selectedCols, ByteArray buf) {
int pos = 0;
- for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
- System.arraycopy(cols[i].array(), cols[i].offset(), buf.array(), buf.offset() + pos, cols[i].length());
- pos += cols[i].length();
+ for (int i = 0; i < selectedCols.trueBitCount(); i++) {
+ int c = selectedCols.trueBitAt(i);
+ System.arraycopy(cols[c].array(), cols[c].offset(), buf.array(), buf.offset() + pos, cols[c].length());
+ pos += cols[c].length();
}
buf.setLength(pos);
}
/** write data to given buffer, like serialize */
- public void exportColumns(BitSet selectedCols, ByteBuffer buf) {
- for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
- buf.put(cols[i].array(), cols[i].offset(), cols[i].length());
+ public void exportColumns(ImmutableBitSet selectedCols, ByteBuffer buf) {
+ for (int i = 0; i < selectedCols.trueBitCount(); i++) {
+ int c = selectedCols.trueBitAt(i);
+ buf.put(cols[c].array(), cols[c].offset(), cols[c].length());
}
}
@@ -232,17 +246,18 @@ public class GTRecord implements Comparable<GTRecord> {
}
/** change pointers to point to data in given buffer, UNLIKE deserialize */
- public void loadColumns(BitSet selectedCols, ByteBuffer buf) {
+ public void loadColumns(ImmutableBitSet selectedCols, ByteBuffer buf) {
int pos = buf.position();
- for (int i = selectedCols.nextSetBit(0); i >= 0; i = selectedCols.nextSetBit(i + 1)) {
- int len = info.codeSystem.codeLength(i, buf);
- cols[i].set(buf.array(), buf.arrayOffset() + pos, len);
+ for (int i = 0; i < selectedCols.trueBitCount(); i++) {
+ int c = selectedCols.trueBitAt(i);
+ int len = info.codeSystem.codeLength(c, buf);
+ cols[c].set(buf.array(), buf.arrayOffset() + pos, len);
pos += len;
buf.position(pos);
}
}
- /* similar to export(primaryKey) but will stop at the first null value */
+ /** similar to export(primaryKey) but will stop at the first null value */
public static ByteArray exportScanKey(GTRecord rec) {
if (rec == null)
return null;
@@ -251,19 +266,20 @@ public class GTRecord implements Comparable<GTRecord> {
BitSet selectedColumns = new BitSet();
int len = 0;
- for (int i = info.primaryKey.nextSetBit(0); i >= 0; i = info.primaryKey.nextSetBit(i + 1)) {
- if (rec.cols[i].array() == null) {
+ for (int i = 0; i < info.primaryKey.trueBitCount(); i++) {
+ int c = info.primaryKey.trueBitAt(i);
+ if (rec.cols[c].array() == null) {
break;
}
- selectedColumns.set(i);
- len += rec.cols[i].length();
+ selectedColumns.set(c);
+ len += rec.cols[c].length();
}
if (selectedColumns.cardinality() == 0)
return null;
ByteArray buf = ByteArray.allocate(len);
- rec.exportColumns(selectedColumns, buf);
+ rec.exportColumns(new ImmutableBitSet(selectedColumns), buf);
return buf;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
index eed5698..6878ef1 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
@@ -4,9 +4,9 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.BitSet;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
public class GTRowBlock {
@@ -110,7 +110,7 @@ public class GTRowBlock {
return new Reader(info.colBlocksAll);
}
- public Reader getReader(BitSet selectedColBlocks) {
+ public Reader getReader(ImmutableBitSet selectedColBlocks) {
return new Reader(selectedColBlocks);
}
@@ -118,9 +118,9 @@ public class GTRowBlock {
int cur;
ByteBuffer primaryKeyBuffer;
ByteBuffer[] cellBlockBuffers;
- BitSet selectedColBlocks;
+ ImmutableBitSet selectedColBlocks;
- Reader(BitSet selectedColBlocks) {
+ Reader(ImmutableBitSet selectedColBlocks) {
primaryKeyBuffer = primaryKey.asBuffer();
cellBlockBuffers = new ByteBuffer[info.colBlocks.length];
for (int i = 0; i < cellBlockBuffers.length; i++) {
@@ -137,7 +137,8 @@ public class GTRowBlock {
if (hasNext() == false)
throw new IllegalArgumentException();
- for (int c = selectedColBlocks.nextSetBit(0); c >= 0; c = selectedColBlocks.nextSetBit(c + 1)) {
+ for (int i = 0; i < selectedColBlocks.trueBitCount(); i++) {
+ int c = selectedColBlocks.trueBitAt(i);
result.loadCellBlock(c, cellBlockBuffers[c]);
}
cur++;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRange.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRange.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRange.java
index 8916f4f..916fb11 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRange.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRange.java
@@ -28,11 +28,12 @@ public class GTScanRange {
private void validateRangeKey(GTRecord pk) {
pk.maskForEqualHashComp(pk.info.primaryKey);
boolean afterNull = false;
- for (int i = pk.info.primaryKey.nextSetBit(0); i >= 0; i = pk.info.primaryKey.nextSetBit(i + 1)) {
+ for (int i = 0; i < pk.info.primaryKey.trueBitCount(); i++) {
+ int c = pk.info.primaryKey.trueBitAt(i);
if (afterNull) {
- pk.cols[i].set(null, 0, 0);
+ pk.cols[c].set(null, 0, 0);
} else {
- afterNull = pk.cols[i].array() == null;
+ afterNull = pk.cols[c].array() == null;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
index 9fddffe..e8565ca 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRangePlanner.java
@@ -1,7 +1,6 @@
package org.apache.kylin.storage.gridtable;
import java.util.ArrayList;
-import java.util.BitSet;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -13,6 +12,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.IFilterCodeSystem;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
@@ -81,8 +81,7 @@ public class GTScanRangePlanner {
pkEnd.set(col, range.end);
if (range.equals != null) {
- BitSet fuzzyMask = new BitSet();
- fuzzyMask.set(col);
+ ImmutableBitSet fuzzyMask = new ImmutableBitSet(col);
for (ByteArray v : range.equals) {
GTRecord fuzzy = new GTRecord(info);
fuzzy.set(col, v);
@@ -473,11 +472,12 @@ public class GTScanRangePlanner {
public int compare(GTRecord a, GTRecord b) {
assert a.info == b.info;
assert a.maskForEqualHashComp() == b.maskForEqualHashComp();
- BitSet mask = a.maskForEqualHashComp();
+ ImmutableBitSet mask = a.maskForEqualHashComp();
int comp = 0;
- for (int i = mask.nextSetBit(0); i >= 0; i = mask.nextSetBit(i + 1)) {
- comp = comparator.compare(a.cols[i], b.cols[i]);
+ for (int i = 0; i < mask.trueBitCount(); i++) {
+ int c = mask.trueBitAt(i);
+ comp = comparator.compare(a.cols[c], b.cols[c]);
if (comp != 0)
return comp;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
index 14af632..1fd3c8e 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTScanRequest.java
@@ -1,9 +1,9 @@
package org.apache.kylin.storage.gridtable;
import java.util.Arrays;
-import java.util.BitSet;
import java.util.Set;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
@@ -14,21 +14,21 @@ public class GTScanRequest {
// basic
private GTInfo info;
private GTScanRange range;
- private BitSet columns;
+ private ImmutableBitSet columns;
// optional filtering
private TupleFilter filterPushDown;
// optional aggregation
- private BitSet aggrGroupBy;
- private BitSet aggrMetrics;
+ private ImmutableBitSet aggrGroupBy;
+ private ImmutableBitSet aggrMetrics;
private String[] aggrMetricsFuncs;
public GTScanRequest(GTInfo info) {
this(info, null, null, null);
}
- public GTScanRequest(GTInfo info, GTScanRange range, BitSet columns, TupleFilter filterPushDown) {
+ public GTScanRequest(GTInfo info, GTScanRange range, ImmutableBitSet columns, TupleFilter filterPushDown) {
this.info = info;
this.range = range == null ? new GTScanRange(new GTRecord(info), new GTRecord(info)) : range;
this.columns = columns;
@@ -36,13 +36,13 @@ public class GTScanRequest {
validate();
}
- public GTScanRequest(GTInfo info, GTScanRange range, BitSet aggrGroupBy, BitSet aggrMetrics, //
+ public GTScanRequest(GTInfo info, GTScanRange range, ImmutableBitSet aggrGroupBy, ImmutableBitSet aggrMetrics, //
String[] aggrMetricsFuncs, TupleFilter filterPushDown) {
this(info, range, null, aggrGroupBy, aggrMetrics, aggrMetricsFuncs, filterPushDown);
}
- public GTScanRequest(GTInfo info, GTScanRange range, BitSet dimensions, BitSet aggrGroupBy, BitSet aggrMetrics, //
- String[] aggrMetricsFuncs, TupleFilter filterPushDown) {
+ public GTScanRequest(GTInfo info, GTScanRange range, ImmutableBitSet dimensions, ImmutableBitSet aggrGroupBy, //
+ ImmutableBitSet aggrMetrics, String[] aggrMetricsFuncs, TupleFilter filterPushDown) {
this.info = info;
this.range = range == null ? new GTScanRange(new GTRecord(info), new GTRecord(info)) : range;
this.columns = dimensions;
@@ -65,15 +65,14 @@ public class GTScanRequest {
if (aggrMetrics.cardinality() != aggrMetricsFuncs.length)
throw new IllegalStateException();
- if (columns == null) {
- columns = new BitSet();
- }
- columns.or(aggrGroupBy);
- columns.or(aggrMetrics);
+ if (columns == null)
+ columns = ImmutableBitSet.EMPTY;
+ columns = columns.or(aggrGroupBy);
+ columns = columns.or(aggrMetrics);
}
if (columns == null)
- columns = (BitSet) info.colAll.clone();
+ columns = info.colAll;
if (hasFilterPushDown()) {
validateFilterPushDown();
@@ -91,7 +90,7 @@ public class GTScanRequest {
// filter columns must belong to the table
info.validateColRef(col);
// filter columns must be returned to satisfy upper layer evaluation (calcite)
- columns.set(col.getColumnDesc().getZeroBasedIndex());
+ columns = columns.set(col.getColumnDesc().getZeroBasedIndex());
}
// un-evaluatable filter must be removed
@@ -102,7 +101,7 @@ public class GTScanRequest {
// columns in un-evaluatable filter must be returned without loss so upper layer can do final evaluation
if (hasAggregation()) {
for (TblColRef col : unevaluableColumns) {
- aggrGroupBy.set(col.getColumnDesc().getZeroBasedIndex());
+ aggrGroupBy = aggrGroupBy.set(col.getColumnDesc().getZeroBasedIndex());
}
}
}
@@ -128,7 +127,7 @@ public class GTScanRequest {
return range.pkEnd;
}
- public BitSet getColumns() {
+ public ImmutableBitSet getColumns() {
return columns;
}
@@ -136,11 +135,11 @@ public class GTScanRequest {
return filterPushDown;
}
- public BitSet getAggrGroupBy() {
+ public ImmutableBitSet getAggrGroupBy() {
return aggrGroupBy;
}
- public BitSet getAggrMetrics() {
+ public ImmutableBitSet getAggrMetrics() {
return aggrMetrics;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
index 76bf76a..cf4a3cc 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
@@ -2,10 +2,9 @@ package org.apache.kylin.storage.gridtable;
import java.io.Closeable;
import java.io.IOException;
-import java.util.BitSet;
import java.util.Iterator;
-import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
public interface IGTStore {
@@ -15,7 +14,7 @@ public interface IGTStore {
IGTStoreWriter append(int shard, GTRowBlock.Writer fillLast) throws IOException;
- IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException;
+ IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, ImmutableBitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException;
interface IGTStoreWriter extends Closeable {
void write(GTRowBlock block) throws IOException;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
index 9647a8b..4dcacf7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
@@ -1,14 +1,24 @@
package org.apache.kylin.storage.gridtable.diskstore;
-import com.google.common.base.Preconditions;
-import org.apache.kylin.storage.gridtable.*;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.storage.gridtable.GTInfo;
+import org.apache.kylin.storage.gridtable.GTRecord;
+import org.apache.kylin.storage.gridtable.GTRowBlock;
+import org.apache.kylin.storage.gridtable.GTScanRequest;
+import org.apache.kylin.storage.gridtable.IGTStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.*;
-import java.nio.ByteBuffer;
-import java.util.BitSet;
-import java.util.UUID;
+import com.google.common.base.Preconditions;
/**
*/
@@ -148,7 +158,7 @@ public class GTDiskStore implements IGTStore {
}
@Override
- public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException {
+ public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, ImmutableBitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException {
return new DiskStoreScanner(fileSystem.getReader(getRowBlockFile(identifier)));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTMemDiskStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTMemDiskStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTMemDiskStore.java
index 45a51ee..65d4905 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTMemDiskStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTMemDiskStore.java
@@ -14,9 +14,9 @@ import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.StandardOpenOption;
-import java.util.BitSet;
import java.util.NoSuchElementException;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.storage.gridtable.GTInfo;
import org.apache.kylin.storage.gridtable.GTRecord;
import org.apache.kylin.storage.gridtable.GTRowBlock;
@@ -89,7 +89,7 @@ public class GTMemDiskStore implements IGTStore, Closeable {
}
@Override
- public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException {
+ public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, ImmutableBitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException {
synchronized (lock) {
return new Reader();
}
@@ -522,9 +522,11 @@ public class GTMemDiskStore implements IGTStore, Closeable {
}
public void activateMemWrite() {
- writeActivated = true;
- if (debug)
- logger.debug(GTMemDiskStore.this + " mem write activated");
+ if (budgetCtrl.getTotalBudgetMB() > 0) {
+ writeActivated = true;
+ if (debug)
+ logger.debug(GTMemDiskStore.this + " mem write activated");
+ }
}
public void deactivateMemWrite() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
index a4d0b8d..b3d77de 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
@@ -2,10 +2,10 @@ package org.apache.kylin.storage.gridtable.memstore;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.BitSet;
import java.util.Iterator;
import java.util.List;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.storage.gridtable.GTInfo;
import org.apache.kylin.storage.gridtable.GTRecord;
import org.apache.kylin.storage.gridtable.GTRowBlock;
@@ -78,7 +78,7 @@ public class GTSimpleMemStore implements IGTStore {
}
@Override
- public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) {
+ public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, ImmutableBitSet selectedColBlocks, GTScanRequest additionalPushDown) {
return new IGTStoreScanner() {
Iterator<GTRowBlock> it = rowBlockList.iterator();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java
index 374f497..d9d1eff 100644
--- a/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/gridtable/DictGridTableTest.java
@@ -12,6 +12,7 @@ import java.util.Map;
import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.NumberDictionaryBuilder;
import org.apache.kylin.dict.StringBytesConverter;
@@ -308,7 +309,7 @@ public class DictGridTableTest {
);
builder.setPrimaryKey(setOf(0, 1));
builder.setColumnPreferIndex(setOf(0));
- builder.enableColumnBlock(new BitSet[] { setOf(0, 1), setOf(2), setOf(3, 4) });
+ builder.enableColumnBlock(new ImmutableBitSet[] { setOf(0, 1), setOf(2), setOf(3, 4) });
builder.enableRowBlock(4);
GTInfo info = builder.build();
return info;
@@ -354,10 +355,10 @@ public class DictGridTableTest {
return builder.build(0);
}
- private static BitSet setOf(int... values) {
+ private static ImmutableBitSet setOf(int... values) {
BitSet set = new BitSet();
for (int i : values)
set.set(i);
- return set;
+ return new ImmutableBitSet(set);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7f5290b3/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java b/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java
index 05e61f8..88db599 100644
--- a/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/gridtable/SimpleGridTableTest.java
@@ -10,6 +10,7 @@ import java.util.List;
import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.storage.gridtable.GTInfo.Builder;
import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore;
@@ -206,7 +207,7 @@ public class SimpleGridTableTest {
static GTInfo advancedInfo() {
Builder builder = infoBuilder();
- builder.enableColumnBlock(new BitSet[] { setOf(0), setOf(1, 2), setOf(3, 4) });
+ builder.enableColumnBlock(new ImmutableBitSet[] { setOf(0), setOf(1, 2), setOf(3, 4) });
builder.enableRowBlock(4);
GTInfo info = builder.build();
return info;
@@ -227,10 +228,10 @@ public class SimpleGridTableTest {
return builder;
}
- private static BitSet setOf(int... values) {
+ private static ImmutableBitSet setOf(int... values) {
BitSet set = new BitSet();
for (int i : values)
set.set(i);
- return set;
+ return new ImmutableBitSet(set);
}
}