You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/10/22 08:07:55 UTC
[3/5] incubator-kylin git commit: temp
temp
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/6f05c14a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/6f05c14a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/6f05c14a
Branch: refs/heads/KYLIN-942
Commit: 6f05c14a715435d049a67f546c4fc5e74b9667ce
Parents: bd56ab3
Author: honma <ho...@ebay.com>
Authored: Mon Oct 19 18:13:53 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Oct 19 18:13:53 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/BitSets.java | 39 +++++++++++++++++
.../org/apache/kylin/common/util/BasicTest.java | 9 ++--
.../apache/kylin/common/util/BitSetsTest.java | 36 ++++++++++++++++
.../apache/kylin/common/util/BytesUtilTest.java | 9 ++--
.../java/org/apache/kylin/gridtable/GTInfo.java | 1 +
.../org/apache/kylin/gridtable/GTRecord.java | 16 +++++++
.../kylin/gridtable/GTScanRangePlanner.java | 12 +-----
.../kylin/query/test/ITKylinQueryTest.java | 4 +-
.../src/test/resources/query/debug/query78.sql | 22 ++++++++++
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 1 +
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 44 +++++++++++++++-----
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 5 ++-
.../hbase/cube/v2/HBaseReadonlyStore.java | 42 ++++++++++++-------
.../kylin/storage/hbase/cube/v2/RawScan.java | 2 +-
.../coprocessor/endpoint/CubeVisitService.java | 2 +-
15 files changed, 195 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6f05c14a/core-common/src/main/java/org/apache/kylin/common/util/BitSets.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BitSets.java b/core-common/src/main/java/org/apache/kylin/common/util/BitSets.java
new file mode 100644
index 0000000..b8a6de7
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/BitSets.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.util.BitSet;
+
+public class BitSets {
+ public static BitSet valueOf(int[] indexes) {
+ if (indexes == null || indexes.length == 0) {
+ return new BitSet();
+ }
+
+ int maxIndex = Integer.MIN_VALUE;
+ for (int index : indexes) {
+ maxIndex = Math.max(maxIndex, index);
+ }
+ BitSet set = new BitSet(maxIndex);
+ for (int index : indexes) {
+ set.set(index);
+ }
+ return set;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6f05c14a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index cd83d96..c8752d7 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -25,7 +25,6 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.IdentityHashMap;
-import java.util.Map;
import org.apache.commons.configuration.ConfigurationException;
import org.junit.Assert;
@@ -33,9 +32,7 @@ import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Predicate;
import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import com.google.common.collect.TreeMultiset;
/**
@@ -101,8 +98,10 @@ public class BasicTest {
public void test1() throws Exception {
System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1433833611000L));
System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1433250517000L));
- System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis("2015-06-01 00:00:00"));
- System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis("2015-05-15 17:00:00"));
+ System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis("2015-10-14 14:40:00"));
+ System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis("2015-10-14 19:00:00"));
+ System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis("2015-10-15 00:00:00"));
+ System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis("2015-10-15 08:45:00"));
String bb = "\\x00\\x00\\x00\\x00\\x01\\x3F\\xD0\\x2D\\58\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00";//2013/07/12 07:59:37
String cc = "\\x00\\x00\\x00\\x00\\x01\\x41\\xBE\\x8F\\xD8\\x00\\x00\\x00\\x00\\x00\\x00\\x00";//2013/10/16 08:00:00
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6f05c14a/core-common/src/test/java/org/apache/kylin/common/util/BitSetsTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BitSetsTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BitSetsTest.java
new file mode 100644
index 0000000..c923969
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BitSetsTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.util.BitSet;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class BitSetsTest {
+
+ @Test
+ public void basicTest() {
+ BitSet a = BitSets.valueOf(new int[] { 1, 3, 10 });
+ Assert.assertEquals(3, a.cardinality());
+ Assert.assertTrue(10 < a.size());
+ Assert.assertTrue(a.get(3));
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6f05c14a/core-common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
index 7436de9..04694ba 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
@@ -18,17 +18,18 @@
package org.apache.kylin.common.util;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
import java.nio.ByteBuffer;
import java.util.Arrays;
-import junit.framework.TestCase;
-
import org.junit.Test;
/**
* by honma
*/
-public class BytesUtilTest extends TestCase {
+public class BytesUtilTest {
@Test
public void test() {
ByteBuffer buffer = ByteBuffer.allocate(10000);
@@ -61,6 +62,7 @@ public class BytesUtilTest extends TestCase {
testWriteReadUnsignedInt(73503300, 4);
}
+ @Test
public void testWriteReadUnsignedInt(int testInt, int length) {
ByteArray ba = new ByteArray(new byte[length]);
BytesUtil.writeUnsigned(testInt, length, ba.asBuffer());
@@ -77,6 +79,7 @@ public class BytesUtilTest extends TestCase {
assertTrue(Arrays.equals(anOtherNewBytes, ba.array()));
}
+ @Test
public void testReadable() {
String x = "\\x00\\x00\\x00\\x00\\x00\\x01\\xFC\\xA8";
byte[] bytes = BytesUtil.fromReadableText(x);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6f05c14a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
index c0ffcc1..e3d3640 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
@@ -38,6 +38,7 @@ public class GTInfo {
private GTInfo() {
}
+
public String getTableName() {
return tableName;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6f05c14a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index 0d02655..d54e223 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -2,6 +2,7 @@ package org.apache.kylin.gridtable;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.List;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
@@ -302,4 +303,19 @@ public class GTRecord implements Comparable<GTRecord> {
}
}
+ /** change pointers to point to data in given buffer, UNLIKE deserialize
+ * unlike loadColumns(ImmutableBitSet selectedCols, ByteBuffer buf), this
+ * method allows to defined specific columns(in order) to load
+ */
+ public void loadColumns(List<Integer> selectedCols, ByteBuffer buf) {
+ int pos = buf.position();
+ for (int i = 0; i < selectedCols.size(); i++) {
+ int c = selectedCols.get(i);
+ int len = info.codeSystem.codeLength(c, buf);
+ cols[c].set(buf.array(), buf.arrayOffset() + pos, len);
+ pos += len;
+ buf.position(pos);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6f05c14a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index 3068225..12c5b05 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -492,11 +492,7 @@ public class GTScanRangePlanner {
@Override
public int compare(ByteArray a, ByteArray b) {
if (a.array() == null) {
- if (b.array() == null) {
- return 0;
- } else {
- return -1;
- }
+ return -1;
} else if (b.array() == null) {
return 1;
} else {
@@ -511,11 +507,7 @@ public class GTScanRangePlanner {
@Override
public int compare(ByteArray a, ByteArray b) {
if (a.array() == null) {
- if (b.array() == null) {
- return 0;
- } else {
- return 1;
- }
+ return 1;
} else if (b.array() == null) {
return -1;
} else {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6f05c14a/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
index 2f53fd7..a594585 100644
--- a/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
@@ -95,7 +95,7 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = "src/test/resources/query/sql/query10.sql";
+ String queryFileName = "src/test/resources/query/sql/query78.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
@@ -124,7 +124,7 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testCommonQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql", null, true);
+ execAndCompQuery("src/test/resources/query/debug", null, true);
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6f05c14a/query/src/test/resources/query/debug/query78.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/debug/query78.sql b/query/src/test/resources/query/debug/query78.sql
new file mode 100644
index 0000000..299f1a4
--- /dev/null
+++ b/query/src/test/resources/query/debug/query78.sql
@@ -0,0 +1,22 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select count(*) as c,sum(PRICE) as GMV, LSTG_FORMAT_NAME as FORMAT_NAME
+from test_kylin_fact
+where (LSTG_FORMAT_NAME in ('ABIN')) or (LSTG_FORMAT_NAME>='FP-GTC' and LSTG_FORMAT_NAME<='Others')
+group by LSTG_FORMAT_NAME
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6f05c14a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index e0f6922..7f683be 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -133,6 +133,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
List<RawScan> rawScans = preparedHBaseScan(scanRequest.getPkStart(), scanRequest.getPkEnd(), scanRequest.getFuzzyKeys(), selectedColBlocks);
+ List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
byte[] scanRequestBytes = KryoUtils.serialize(scanRequest);
final ByteString scanRequestBytesString = ByteString.copyFrom(scanRequestBytes);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6f05c14a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 91228cf..1d217ac 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -154,36 +154,60 @@ public abstract class CubeHBaseRPC {
return buf;
}
+ /**
+ * prune untouched hbase columns
+ */
protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) {
List<Pair<byte[], byte[]>> result = Lists.newArrayList();
- int colBlockIdx = 1; // start from 1; the 0th column block is primary key which maps to rowkey
+ int colBlkIndex = 1;
HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
byte[] byteFamily = Bytes.toBytes(familyDesc.getName());
for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
- if (selectedColBlocks.get(colBlockIdx)) {
+ if (selectedColBlocks.get(colBlkIndex)) {
byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier());
result.add(new Pair<byte[], byte[]>(byteFamily, byteQualifier));
}
- colBlockIdx++;
+ colBlkIndex++;
}
}
return result;
}
- //possible to use binary search as cells might be sorted
- public static Cell findCell(List<Cell> cells, byte[] familyName, byte[] columnName) {
- for (Cell c : cells) {
- if (BytesUtil.compareBytes(familyName, 0, c.getFamilyArray(), c.getFamilyOffset(), familyName.length) == 0 && //
- BytesUtil.compareBytes(columnName, 0, c.getQualifierArray(), c.getQualifierOffset(), columnName.length) == 0) {
- return c;
+ /**
+ * for each selected hbase column, it might contain values of multiple GT columns.
+ * The mapping should be passed down to storage
+ */
+ protected List<List<Integer>> getHBaseColumnsGTMapping(ImmutableBitSet selectedColBlocks) {
+
+ List<List<Integer>> ret = Lists.newArrayList();
+
+ int colBlkIndex = 1;
+ int metricOffset = fullGTInfo.getPrimaryKey().trueBitCount();
+
+ HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
+ for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
+ for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
+ if (selectedColBlocks.get(colBlkIndex)) {
+ int[] metricIndexes = hbaseColDesc.getMeasureIndex();
+ Integer[] gtIndexes = new Integer[metricIndexes.length];
+ for (int i = 0; i < gtIndexes.length; i++) {
+ gtIndexes[i] = metricIndexes[i] + metricOffset;
+ }
+ ret.add(Arrays.asList(gtIndexes));
+ }
+ colBlkIndex++;
}
}
- return null;
+
+ Preconditions.checkState(selectedColBlocks.trueBitCount() == ret.size() + 1);
+ return ret;
}
+
+
public static void applyHBaseColums(Scan scan, List<Pair<byte[], byte[]>> hbaseColumns) {
for (Pair<byte[], byte[]> hbaseColumn : hbaseColumns) {
byte[] byteFamily = hbaseColumn.getFirst();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6f05c14a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index a31bcdf..56e6c5c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -74,12 +74,13 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
List<RawScan> rawScans = preparedHBaseScan(scanRequest.getPkStart(), scanRequest.getPkEnd(), scanRequest.getFuzzyKeys(), selectedColBlocks);
+ List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
final List<ResultScanner> scanners = Lists.newArrayList();
final List<Iterator<Result>> resultIterators = Lists.newArrayList();
for (RawScan rawScan : rawScans) {
-
+
logScan(rawScan, cubeSeg.getStorageLocationIdentifier());
Scan hbaseScan = buildScan(rawScan);
@@ -117,7 +118,7 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
}
};
- IGTStore store = new HBaseReadonlyStore(cellListIterator, scanRequest, rawScans.get(0).hbaseColumns);
+ IGTStore store = new HBaseReadonlyStore(cellListIterator, scanRequest, rawScans.get(0).hbaseColumns, hbaseColumnsToGT);
IGTScanner rawScanner = store.scan(scanRequest);
final IGTScanner decorateScanner = scanRequest.decorateScanner(rawScanner);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6f05c14a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
index 04935cf..d0551bb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
@@ -24,7 +24,7 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.gridtable.GTInfo;
@@ -34,20 +34,22 @@ import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStore;
import org.apache.kylin.gridtable.IGTWriter;
+import com.google.common.base.Preconditions;
+
public class HBaseReadonlyStore implements IGTStore {
private CellListIterator cellListIterator;
private GTInfo info;
private List<Pair<byte[], byte[]>> hbaseColumns;
- private ImmutableBitSet selectedColBlocks;
+ private List<List<Integer>> hbaseColumnsToGT;
- public HBaseReadonlyStore(CellListIterator cellListIterator, GTScanRequest gtScanRequest, List<Pair<byte[], byte[]>> hbaseColumns) {
+ public HBaseReadonlyStore(CellListIterator cellListIterator, GTScanRequest gtScanRequest, List<Pair<byte[], byte[]>> hbaseColumns, List<List<Integer>> hbaseColumnsToGT) {
this.cellListIterator = cellListIterator;
this.info = gtScanRequest.getInfo();
this.hbaseColumns = hbaseColumns;
- this.selectedColBlocks = gtScanRequest.getSelectedColBlocks().set(0);
+ this.hbaseColumnsToGT = hbaseColumnsToGT;
}
@Override
@@ -65,11 +67,22 @@ public class HBaseReadonlyStore implements IGTStore {
throw new UnsupportedOperationException();
}
+ //TODO: possible to use binary search as cells might be sorted?
+ public static Cell findCell(List<Cell> cells, byte[] familyName, byte[] columnName) {
+ for (Cell c : cells) {
+ if (BytesUtil.compareBytes(familyName, 0, c.getFamilyArray(), c.getFamilyOffset(), familyName.length) == 0 && //
+ BytesUtil.compareBytes(columnName, 0, c.getQualifierArray(), c.getQualifierOffset(), columnName.length) == 0) {
+ return c;
+ }
+ }
+ return null;
+ }
+
@Override
public IGTScanner scan(GTScanRequest scanRequest) throws IOException {
return new IGTScanner() {
int count;
-
+
@Override
public void close() throws IOException {
cellListIterator.close();
@@ -79,7 +92,7 @@ public class HBaseReadonlyStore implements IGTStore {
public Iterator<GTRecord> iterator() {
return new Iterator<GTRecord>() {
GTRecord oneRecord = new GTRecord(info); // avoid object creation
-
+
@Override
public boolean hasNext() {
return cellListIterator.hasNext();
@@ -93,20 +106,19 @@ public class HBaseReadonlyStore implements IGTStore {
}
ByteBuffer buf;
-
+
// dimensions, set to primary key, also the 0th column block
Cell firstCell = oneRow.get(0);
- buf = byteBuffer(firstCell.getRowArray(), RowConstants.ROWKEY_HEADER_LEN + firstCell.getRowOffset(), firstCell.getRowLength() - RowConstants.ROWKEY_CUBOIDID_LEN);
+ buf = byteBuffer(firstCell.getRowArray(), RowConstants.ROWKEY_HEADER_LEN + firstCell.getRowOffset(), firstCell.getRowLength() - RowConstants.ROWKEY_HEADER_LEN);
oneRecord.loadCellBlock(0, buf);
// metrics
- int hbaseColIdx = 0;
- for (int i = 1; i < selectedColBlocks.trueBitCount(); i++) {
- int colBlockIdx = selectedColBlocks.trueBitAt(i);
- Pair<byte[], byte[]> hbaseColumn = hbaseColumns.get(hbaseColIdx++);
- Cell cell = CubeHBaseRPC.findCell(oneRow, hbaseColumn.getFirst(), hbaseColumn.getSecond());
+ for (int i = 0; i < hbaseColumns.size(); i++) {
+ Pair<byte[], byte[]> hbaseColumn = hbaseColumns.get(i);
+ Cell cell = HBaseReadonlyStore.findCell(oneRow, hbaseColumn.getFirst(), hbaseColumn.getSecond());
+ Preconditions.checkNotNull(cell);
buf = byteBuffer(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
- oneRecord.loadCellBlock(colBlockIdx, buf);
+ oneRecord.loadColumns(hbaseColumnsToGT.get(i), buf);
}
return oneRecord;
@@ -116,7 +128,7 @@ public class HBaseReadonlyStore implements IGTStore {
public void remove() {
throw new UnsupportedOperationException();
}
-
+
private ByteBuffer byteBuffer(byte[] array, int offset, int length) {
return ByteBuffer.wrap(array, offset, length);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6f05c14a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
index 0184908..ad4263f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
@@ -27,7 +27,7 @@ public class RawScan {
public byte[] startKey;
public byte[] endKey;
- public List<Pair<byte[], byte[]>> hbaseColumns;
+ public List<Pair<byte[], byte[]>> hbaseColumns;//only contain interested columns
public List<Pair<byte[], byte[]>> fuzzyKey;
public RawScan(byte[] startKey, byte[] endKey, List<Pair<byte[], byte[]>> hbaseColumns, List<Pair<byte[], byte[]>> fuzzyKey) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6f05c14a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index f0b8c6f..c4927ab 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -136,7 +136,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
innerScanner = region.getScanner(scan);
InnerScannerAsIterator cellListIterator = new InnerScannerAsIterator(innerScanner);
- IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScan.hbaseColumns);
+ IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScan.hbaseColumns,null);
IGTScanner rawScanner = store.scan(scanReq);
IGTScanner finalScanner = scanReq.decorateScanner(rawScanner);