You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/03/04 09:01:24 UTC
[1/2] kylin git commit: KYLIN-1471 - LIMIT after having clause should
not be pushed down to storage context
Repository: kylin
Updated Branches:
refs/heads/2.x-staging b41c44600 -> 2f44970d2
KYLIN-1471 - LIMIT after having clause should not be pushed down to storage context
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2f44970d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2f44970d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2f44970d
Branch: refs/heads/2.x-staging
Commit: 2f44970d28318a21c16aaa8f28e844c5d88f5e3d
Parents: b9a3418
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 4 16:00:11 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 4 16:01:04 2016 +0800
----------------------------------------------------------------------
.../resources/query/sql_tableau/query29.sql | 29 ++++++++++++++++++++
.../apache/kylin/query/relnode/OLAPContext.java | 1 +
.../kylin/query/relnode/OLAPFilterRel.java | 2 ++
.../kylin/query/relnode/OLAPLimitRel.java | 21 ++++++++------
4 files changed, 44 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2f44970d/kylin-it/src/test/resources/query/sql_tableau/query29.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_tableau/query29.sql b/kylin-it/src/test/resources/query/sql_tableau/query29.sql
new file mode 100644
index 0000000..0858087
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_tableau/query29.sql
@@ -0,0 +1,29 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT *
+ FROM (
+ select test_kylin_fact.lstg_format_name, test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
+ , count(*) as TRANS_CNT
+ from test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ where test_cal_dt.week_beg_dt between DATE '2013-05-01' and DATE '2013-08-01'
+ group by test_kylin_fact.lstg_format_name, test_cal_dt.week_beg_dt
+ ) "TableauSQL"
+ LIMIT 1
http://git-wip-us.apache.org/repos/asf/kylin/blob/2f44970d/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 6de1790..431328f 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -103,6 +103,7 @@ public class OLAPContext {
public OLAPTableScan firstTableScan = null; // to be fact table scan except "select * from lookupTable"
public TupleInfo returnTupleInfo = null;
public boolean afterAggregate = false;
+ public boolean afterSkippedFilter = false;
public boolean afterJoin = false;
public boolean hasJoin = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/2f44970d/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 7b8bfdb..a847890 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -271,6 +271,8 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
// only translate where clause and don't translate having clause
if (!context.afterAggregate) {
translateFilter(context);
+ } else {
+ context.afterSkippedFilter = true;//having clause is skipped
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2f44970d/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index 572a5c7..82aa9de 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -73,16 +73,19 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel {
implementor.visitChild(getInput(), this);
this.columnRowType = buildColumnRowType();
-
this.context = implementor.getContext();
- Number limitValue = (Number) (((RexLiteral) localFetch).getValue());
- int limit = limitValue.intValue();
- this.context.storageContext.setLimit(limit);
- this.context.limit = limit;
- if(localOffset != null) {
- Number offsetValue = (Number) (((RexLiteral) localOffset).getValue());
- int offset = offsetValue.intValue();
- this.context.storageContext.setOffset(offset);
+
+ if (!context.afterSkippedFilter) {
+ Number limitValue = (Number) (((RexLiteral) localFetch).getValue());
+ int limit = limitValue.intValue();
+ this.context.storageContext.setLimit(limit);
+ this.context.limit = limit;
+
+ if (localOffset != null) {
+ Number offsetValue = (Number) (((RexLiteral) localOffset).getValue());
+ int offset = offsetValue.intValue();
+ this.context.storageContext.setOffset(offset);
+ }
}
}
[2/2] kylin git commit: KYLIN-1465 log improvement on others
Posted by ma...@apache.org.
KYLIN-1465 log improvement on others
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b9a3418b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b9a3418b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b9a3418b
Branch: refs/heads/2.x-staging
Commit: b9a3418b7f2121745090bb6463e7be754babfb73
Parents: b41c446
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 4 15:59:37 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 4 16:01:04 2016 +0800
----------------------------------------------------------------------
.../kylin/cube/gridtable/CubeGridTable.java | 6 ++---
.../gridtable/NotEnoughGTInfoException.java | 23 --------------------
.../kylin/gridtable/GTScanReqSerDerTest.java | 11 +++++-----
.../kylin/storage/translate/HBaseKeyRange.java | 4 +---
.../streaming/cube/StreamingCubeBuilder.java | 6 +++--
.../hbase/cube/v1/CubeSegmentTupleIterator.java | 14 ++++++------
.../storage/hbase/cube/v1/CubeStorageQuery.java | 6 ++---
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 18 +++++++--------
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 8 +++----
.../hbase/cube/v2/CubeSegmentScanner.java | 3 +--
.../storage/hbase/cube/v2/CubeStorageQuery.java | 10 +++------
.../kylin/storage/hbase/cube/v2/RawScan.java | 2 +-
12 files changed, 41 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
index 05fc8a5..5f0bb07 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
@@ -29,18 +29,18 @@ public class CubeGridTable {
dictionaryMap.put(col, dictionary);
}
}
-
+
return dictionaryMap;
}
- public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) throws NotEnoughGTInfoException {
+ public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) {
Map<TblColRef, Dictionary<String>> dictionaryMap = getDimensionToDictionaryMap(cubeSeg, cuboidId);
Cuboid cuboid = Cuboid.findById(cubeSeg.getCubeDesc(), cuboidId);
for (TblColRef dim : cuboid.getColumns()) {
if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(dim)) {
Dictionary dict = dictionaryMap.get(dim);
if (dict == null) {
- throw new NotEnoughGTInfoException();
+ throw new RuntimeException("Dictionary for " + dim + " is not found");
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
deleted file mode 100644
index 9bbcf75..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.gridtable;
-
-@SuppressWarnings("serial")
-public class NotEnoughGTInfoException extends Exception {
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
index 6642d95..77cc2d8 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
@@ -29,7 +29,6 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.gridtable.CubeGridTable;
-import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -73,7 +72,7 @@ public class GTScanReqSerDerTest extends LocalFileMetadataTestCase {
buffer.flip();
GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
- this.compareTwoGTInfo(info,sInfo);
+ this.compareTwoGTInfo(info, sInfo);
}
@Test
@@ -83,11 +82,11 @@ public class GTScanReqSerDerTest extends LocalFileMetadataTestCase {
buffer.flip();
GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
- this.compareTwoGTInfo(info,sInfo);
+ this.compareTwoGTInfo(info, sInfo);
}
@Test
- public void testGTInfo() throws NotEnoughGTInfoException {
+ public void testGTInfo() {
CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("test_kylin_cube_with_slr_ready");
CubeSegment segment = cube.getFirstSegment();
@@ -95,8 +94,8 @@ public class GTScanReqSerDerTest extends LocalFileMetadataTestCase {
GTInfo.serializer.serialize(info, buffer);
buffer.flip();
- GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
- this.compareTwoGTInfo(info,sInfo);
+ GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
+ this.compareTwoGTInfo(info, sInfo);
}
private void compareTwoGTInfo(GTInfo info, GTInfo sInfo) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
index a5ca62b..4f39cf6 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
@@ -45,9 +45,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
/**
- *
* @author xjiang
- *
*/
public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
@@ -147,7 +145,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
buf.append(BytesUtil.toHex(fuzzyKey.getFirst()));
buf.append(" ");
buf.append(BytesUtil.toHex(fuzzyKey.getSecond()));
- buf.append(System.lineSeparator());
+ buf.append(";");
}
this.fuzzyKeyString = buf.toString();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
index d7056cf..c4f2b7e 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
@@ -73,6 +73,7 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
private static final Logger logger = LoggerFactory.getLogger(StreamingCubeBuilder.class);
private final String cubeName;
+ private int processedRowCount = 0;
public StreamingCubeBuilder(String cubeName) {
this.cubeName = cubeName;
@@ -81,16 +82,16 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
@Override
public void build(StreamingBatch streamingBatch, Map<TblColRef, Dictionary<String>> dictionaryMap, ICuboidWriter cuboidWriter) {
try {
-
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
final CubeInstance cubeInstance = cubeManager.reloadCubeLocal(cubeName);
LinkedBlockingQueue<List<String>> blockingQueue = new LinkedBlockingQueue<List<String>>();
InMemCubeBuilder inMemCubeBuilder = new InMemCubeBuilder(cubeInstance.getDescriptor(), dictionaryMap);
final Future<?> future = Executors.newCachedThreadPool().submit(inMemCubeBuilder.buildAsRunnable(blockingQueue, cuboidWriter));
+ processedRowCount = streamingBatch.getMessages().size();
for (StreamingMessage streamingMessage : streamingBatch.getMessages()) {
blockingQueue.put(streamingMessage.getData());
}
- blockingQueue.put(Collections.<String> emptyList());
+ blockingQueue.put(Collections.<String>emptyList());
future.get();
cuboidWriter.flush();
@@ -157,6 +158,7 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
public void commit(IBuildable buildable) {
CubeSegment cubeSegment = (CubeSegment) buildable;
cubeSegment.setStatus(SegmentStatusEnum.READY);
+ cubeSegment.setInputRecords(processedRowCount);
CubeUpdate cubeBuilder = new CubeUpdate(cubeSegment.getCubeInstance());
cubeBuilder.setToAddSegs(cubeSegment);
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
index 5e842f7..909de39 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
@@ -210,26 +210,26 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
private void logScan(HBaseKeyRange keyRange) {
StringBuilder info = new StringBuilder();
- info.append("\nScan hbase table ").append(tableName).append(": ");
+ info.append(" Scan hbase table ").append(tableName).append(": ");
if (keyRange.getCuboid().requirePostAggregation()) {
- info.append("cuboid require post aggregation, from ");
+ info.append(" cuboid require post aggregation, from ");
} else {
- info.append("cuboid exact match, from ");
+ info.append(" cuboid exact match, from ");
}
info.append(keyRange.getCuboid().getInputID());
info.append(" to ");
info.append(keyRange.getCuboid().getId());
- info.append("\nStart: ");
+ info.append(" Start: ");
info.append(keyRange.getStartKeyAsString());
info.append(" - ");
info.append(Bytes.toStringBinary(keyRange.getStartKey()));
- info.append("\nStop: ");
+ info.append(" Stop: ");
info.append(keyRange.getStopKeyAsString());
info.append(" - ");
info.append(Bytes.toStringBinary(keyRange.getStopKey()));
if (this.scan.getFilter() != null) {
- info.append("\nFuzzy key counts: " + keyRange.getFuzzyKeys().size());
- info.append("\nFuzzy: ");
+ info.append(" Fuzzy key counts: " + keyRange.getFuzzyKeys().size());
+ info.append(" Fuzzy: ");
info.append(keyRange.getFuzzyKeyAsString());
}
logger.info(info.toString());
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 1b8b586..3d7f620 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -493,7 +493,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
List<Collection<ColumnValueRange>> result = Lists.newArrayList();
if (flatFilter == null) {
- result.add(Collections.<ColumnValueRange> emptyList());
+ result.add(Collections.<ColumnValueRange>emptyList());
return result;
}
@@ -535,7 +535,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
}
if (globalAlwaysTrue) {
orAndRanges.clear();
- orAndRanges.add(Collections.<ColumnValueRange> emptyList());
+ orAndRanges.add(Collections.<ColumnValueRange>emptyList());
}
return orAndRanges;
}
@@ -762,7 +762,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
private void setLimit(TupleFilter filter, StorageContext context) {
boolean goodAggr = context.isExactAggregation();
boolean goodFilter = filter == null || (TupleFilter.isEvaluableRecursively(filter) && context.isCoprocessorEnabled());
- boolean goodSort = context.hasSort() == false;
+ boolean goodSort = !context.hasSort();
if (goodAggr && goodFilter && goodSort) {
logger.info("Enable limit " + context.getLimit());
context.enableLimit();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index f22964f..e6f9ac1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -293,15 +293,15 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final AtomicInteger totalScannedCount = new AtomicInteger(0);
final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(scanRequests.size() * shardNum);
+ final String currentThreadName = Thread.currentThread().getName();
for (final Pair<byte[], byte[]> epRange : getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) {
executorService.submit(new Runnable() {
@Override
public void run() {
for (int i = 0; i < scanRequests.size(); ++i) {
- int scanIndex = i;
CubeVisitProtos.CubeVisitRequest.Builder builder = CubeVisitProtos.CubeVisitRequest.newBuilder();
- builder.setGtScanRequest(scanRequestByteStrings.get(scanIndex)).setHbaseRawScan(rawScanByteStrings.get(scanIndex));
+ builder.setGtScanRequest(scanRequestByteStrings.get(i)).setHbaseRawScan(rawScanByteStrings.get(i));
for (IntList intList : hbaseColumnsToGTIntList) {
builder.addHbaseColumnsToGT(intList);
}
@@ -317,7 +317,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
for (Map.Entry<byte[], CubeVisitProtos.CubeVisitResponse> result : results.entrySet()) {
totalScannedCount.addAndGet(result.getValue().getStats().getScannedRowCount());
- logger.info(getStatsString(result));
+ logger.info("<spawned by " + currentThreadName + ">" + getStatsString(result));
try {
epResultItr.append(CompressionUtils.decompress(HBaseZeroCopyByteString.zeroCopyGetBytes(result.getValue().getCompressedRows())));
} catch (IOException | DataFormatException e) {
@@ -335,12 +335,12 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
private String getStatsString(Map.Entry<byte[], CubeVisitProtos.CubeVisitResponse> result) {
StringBuilder sb = new StringBuilder();
Stats stats = result.getValue().getStats();
- sb.append("Endpoint RPC returned from HTable " + cubeSeg.getStorageLocationIdentifier() + " Shard " + BytesUtil.toHex(result.getKey()) + " on host: " + stats.getHostname() + ".");
- sb.append("Total scanned row: " + stats.getScannedRowCount() + ". ");
- sb.append("Total filtered/aggred row: " + stats.getAggregatedRowCount() + ". ");
- sb.append("Time elapsed in EP: " + (stats.getServiceEndTime() - stats.getServiceStartTime()) + "(ms). ");
- sb.append("Server CPU usage: " + stats.getSystemCpuLoad() + ", server physical mem left: " + stats.getFreePhysicalMemorySize() + ", server swap mem left:" + stats.getFreeSwapSpaceSize() + ".");
- sb.append("Etc message: " + stats.getEtcMsg() + ".");
+ sb.append("Endpoint RPC returned from HTable ").append(cubeSeg.getStorageLocationIdentifier()).append(" Shard ").append(BytesUtil.toHex(result.getKey())).append(" on host: ").append(stats.getHostname()).append(".");
+ sb.append("Total scanned row: ").append(stats.getScannedRowCount()).append(". ");
+ sb.append("Total filtered/aggred row: ").append(stats.getAggregatedRowCount()).append(". ");
+ sb.append("Time elapsed in EP: ").append(stats.getServiceEndTime() - stats.getServiceStartTime()).append("(ms). ");
+ sb.append("Server CPU usage: ").append(stats.getSystemCpuLoad()).append(", server physical mem left: ").append(stats.getFreePhysicalMemorySize()).append(", server swap mem left:").append(stats.getFreeSwapSpaceSize()).append(".");
+ sb.append("Etc message: ").append(stats.getEtcMsg()).append(".");
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index db39455..eb5ac9f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -273,20 +273,20 @@ public abstract class CubeHBaseRPC {
info.append(cuboid.getInputID());
info.append(" to ");
info.append(cuboid.getId());
- info.append("\nStart: ");
+ info.append(" Start: ");
info.append(rawScan.getStartKeyAsString());
info.append(" (");
info.append(Bytes.toStringBinary(rawScan.startKey) + ")");
- info.append("\nStop: ");
+ info.append(" Stop: ");
info.append(rawScan.getEndKeyAsString());
info.append(" (");
info.append(Bytes.toStringBinary(rawScan.endKey) + ")");
if (rawScan.fuzzyKeys != null && rawScan.fuzzyKeys.size() != 0) {
- info.append("\nFuzzy key counts: " + rawScan.fuzzyKeys.size());
+ info.append(" Fuzzy key counts: " + rawScan.fuzzyKeys.size());
info.append(". Fuzzy keys : ");
info.append(rawScan.getFuzzyKeyAsString());
} else {
- info.append("\nNo Fuzzy Key");
+ info.append(", No Fuzzy Key");
}
logger.info(info.toString());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index 3f00566..ee5ae96 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -21,7 +21,6 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.gridtable.CubeGridTable;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
-import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.dict.TupleFilterFunctionTransformer;
import org.apache.kylin.gridtable.EmptyGTScanner;
@@ -56,7 +55,7 @@ public class CubeSegmentScanner implements IGTScanner {
final Cuboid cuboid;
public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, //
- Collection<FunctionDesc> metrics, TupleFilter filter, boolean allowPreAggregate) throws NotEnoughGTInfoException {
+ Collection<FunctionDesc> metrics, TupleFilter filter, boolean allowPreAggregate) {
this.cuboid = cuboid;
this.cubeSeg = cubeSeg;
this.info = CubeGridTable.newGTInfo(cubeSeg, cuboid.getId());
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index ab8c80f..df0bb84 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -11,7 +11,6 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
@@ -96,14 +95,11 @@ public class CubeStorageQuery implements ICachableStorageQuery {
List<CubeSegmentScanner> scanners = Lists.newArrayList();
for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
CubeSegmentScanner scanner;
- try {
- scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, !isExactAggregation);
- } catch (NotEnoughGTInfoException e) {
- //deal with empty cube segment
- logger.info("Cannot construct Segment {}'s GTInfo, this may due to empty segment or broken metadata", cubeSeg);
- logger.info("error stack", e);
+ if (cubeSeg.getInputRecords() == 0) {
+ logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
continue;
}
+ scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, !isExactAggregation);
scanners.add(scanner);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
index 361b1dd..c2ffdba 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
@@ -61,7 +61,7 @@ public class RawScan {
buf.append(BytesUtil.toHex(fuzzyKey.getFirst()));
buf.append(" ");
buf.append(BytesUtil.toHex(fuzzyKey.getSecond()));
- buf.append(System.lineSeparator());
+ buf.append(";");
}
return buf.toString();
}