You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/10/26 10:00:34 UTC
incubator-kylin git commit: KYLIN-942 fix bug when dealing with empty
segment
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-942-x [created] ddeeb09de
KYLIN-942 fix bug when dealing with empty segment
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ddeeb09d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ddeeb09d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ddeeb09d
Branch: refs/heads/KYLIN-942-x
Commit: ddeeb09de04d3ecf9eb6bbc70c57a255ab492eba
Parents: 06aeb99
Author: honma <ho...@ebay.com>
Authored: Mon Oct 26 17:04:30 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Oct 26 17:04:30 2015 +0800
----------------------------------------------------------------------
.../kylin/cube/gridtable/CubeCodeSystem.java | 8 +++---
.../kylin/cube/gridtable/CubeGridTable.java | 15 ++++++++--
.../gridtable/NotEnoughGTInfoException.java | 22 +++++++++++++++
.../org/apache/kylin/gridtable/GTScanRange.java | 17 ++++++++++++
query/src/test/resources/query/sql/query83.sql | 29 --------------------
.../resources/query/sql/query83.sql.disable | 29 ++++++++++++++++++++
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 1 +
.../hbase/cube/v2/CubeSegmentScanner.java | 6 ++--
.../storage/hbase/cube/v2/CubeStorageQuery.java | 10 ++++++-
.../kylin/storage/hbase/cube/v2/HBaseScan.java | 2 +-
10 files changed, 100 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ddeeb09d/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index 9e45fe0..427568c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -69,11 +69,11 @@ public class CubeCodeSystem implements IGTCodeSystem {
serializers = new DataTypeSerializer[info.getColumnCount()];
for (int i = 0; i < info.getColumnCount(); i++) {
// dimension with dictionary
- if (dictionaryMap.get(i) != null) {
+ if (dictionaryMap.containsKey(i)) {
serializers[i] = new DictionarySerializer(dictionaryMap.get(i));
}
// dimension of fixed length
- else if (fixLenMap.get(i) != null) {
+ else if (fixLenMap.containsKey(i)) {
serializers[i] = new FixLenSerializer(fixLenMap.get(i));
}
// metrics
@@ -229,7 +229,7 @@ public class CubeCodeSystem implements IGTCodeSystem {
public int getStorageBytesEstimate() {
return dictionary.getSizeOfId();
}
-
+
@Override
public Object valueOf(byte[] value) {
throw new UnsupportedOperationException();
@@ -305,7 +305,7 @@ public class CubeCodeSystem implements IGTCodeSystem {
public int getStorageBytesEstimate() {
return fixLen;
}
-
+
@Override
public Object valueOf(byte[] value) {
try {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ddeeb09d/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
index 2ee5992..93654f8 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
@@ -11,6 +11,7 @@ import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.metadata.model.TblColRef;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
@SuppressWarnings("rawtypes")
@@ -32,8 +33,18 @@ public class CubeGridTable {
return dictionaryMap;
}
- public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) {
+ public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) throws NotEnoughGTInfoException {
Map<TblColRef, Dictionary<?>> dictionaryMap = getDimensionToDictionaryMap(cubeSeg, cuboidId);
+ Cuboid cuboid = Cuboid.findById(cubeSeg.getCubeDesc(), cuboidId);
+ for (TblColRef dim : cuboid.getColumns()) {
+ if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(dim)) {
+ Dictionary dict = dictionaryMap.get(dim);
+ if (dict == null) {
+ throw new NotEnoughGTInfoException();
+ }
+ }
+ }
+
return newGTInfo(cubeSeg.getCubeDesc(), cuboidId, dictionaryMap);
}
@@ -48,6 +59,7 @@ public class CubeGridTable {
int colIndex = mapping.getIndexOf(dim);
if (cubeDesc.getRowkey().isUseDictionary(dim)) {
Dictionary dict = dictionaryMap.get(dim);
+ Preconditions.checkState(dict != null);
dictionaryByColIdx.put(colIndex, dict);
} else {
int len = cubeDesc.getRowkey().getColumnLength(dim);
@@ -66,5 +78,4 @@ public class CubeGridTable {
builder.enableColumnBlock(mapping.getColumnBlocks());
return builder.build();
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ddeeb09d/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
new file mode 100644
index 0000000..045b11e
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.gridtable;
+
+public class NotEnoughGTInfoException extends Exception {
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ddeeb09d/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
index eefe88e..d5fa6c0 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
@@ -3,6 +3,11 @@ package org.apache.kylin.gridtable;
import java.util.Collections;
import java.util.List;
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+
public class GTScanRange {
final public GTRecord pkStart; // inclusive, record must not be null, col[pk].array() can be null to mean unbounded
@@ -22,6 +27,18 @@ public class GTScanRange {
this.fuzzyKeys = fuzzyKeys == null ? Collections.<GTRecord> emptyList() : fuzzyKeys;
}
+ public GTScanRange replaceGTInfo(final GTInfo gtInfo) {
+ return new GTScanRange(new GTRecord(gtInfo, pkStart.maskForEqualHashComp(), pkStart.cols), //
+ new GTRecord(gtInfo, pkEnd.maskForEqualHashComp(), pkEnd.cols), //
+ Lists.transform(fuzzyKeys, new Function<GTRecord, GTRecord>() {
+ @Nullable
+ @Override
+ public GTRecord apply(GTRecord input) {
+ return new GTRecord(gtInfo, input.maskForEqualHashComp(), input.cols);
+ }
+ }));
+ }
+
@Override
public int hashCode() {
final int prime = 31;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ddeeb09d/query/src/test/resources/query/sql/query83.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query83.sql b/query/src/test/resources/query/sql/query83.sql
deleted file mode 100644
index 514beb4..0000000
--- a/query/src/test/resources/query/sql/query83.sql
+++ /dev/null
@@ -1,29 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT
- seller_id
- FROM test_kylin_fact
- inner JOIN edw.test_cal_dt as test_cal_dt
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- inner JOIN test_category_groupings
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
- AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- where test_kylin_fact.cal_dt < DATE '2013-02-01'
- group by
- test_kylin_fact.seller_id order by sum(test_kylin_fact.price) desc limit 20
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ddeeb09d/query/src/test/resources/query/sql/query83.sql.disable
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query83.sql.disable b/query/src/test/resources/query/sql/query83.sql.disable
new file mode 100644
index 0000000..514beb4
--- /dev/null
+++ b/query/src/test/resources/query/sql/query83.sql.disable
@@ -0,0 +1,29 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+ seller_id
+ FROM test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
+ AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ where test_kylin_fact.cal_dt < DATE '2013-02-01'
+ group by
+ test_kylin_fact.seller_id order by sum(test_kylin_fact.price) desc limit 20
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ddeeb09d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 8557da9..5674986 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -145,6 +145,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
byte[] scanRequestBytes = KryoUtils.serialize(scanRequest);
final ByteString scanRequestBytesString = HBaseZeroCopyByteString.wrap(scanRequestBytes);
+ logger.info("Serialized scanRequestBytes's size is " + scanRequestBytes.length);
ExecutorService executorService = Executors.newFixedThreadPool(rawScans.size());
final List<byte[]> rowBlocks = Collections.synchronizedList(Lists.<byte[]> newArrayList());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ddeeb09d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index 286da55..814a359 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -19,6 +19,7 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.gridtable.CubeGridTable;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
+import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
@@ -46,7 +47,7 @@ public class CubeSegmentScanner implements IGTScanner {
final Cuboid cuboid;
public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, //
- Collection<FunctionDesc> metrics, TupleFilter filter, boolean allowPreAggregate) {
+ Collection<FunctionDesc> metrics, TupleFilter filter, boolean allowPreAggregate) throws NotEnoughGTInfoException {
this.cuboid = cuboid;
this.cubeSeg = cubeSeg;
this.info = CubeGridTable.newGTInfo(cubeSeg, cuboid.getId());
@@ -81,7 +82,8 @@ public class CubeSegmentScanner implements IGTScanner {
GTInfo trimmedInfo = GTInfo.deserialize(trimmedInfoBytes);
for (GTScanRange range : scanRanges) {
- scanRequests.add(new GTScanRequest(trimmedInfo, range, gtDimensions, gtAggrGroups, gtAggrMetrics, gtAggrFuncs, gtFilter, allowPreAggregate));
+ scanRequests.add(new GTScanRequest(trimmedInfo, range,//range.replaceGTInfo(trimmedInfo),
+ gtDimensions, gtAggrGroups, gtAggrMetrics, gtAggrFuncs, gtFilter, allowPreAggregate));
}
scanner = new Scanner();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ddeeb09d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index 71abb41..1d9ee3a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -11,6 +11,7 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
@@ -96,7 +97,14 @@ public class CubeStorageQuery implements ICachableStorageQuery {
List<CubeSegmentScanner> scanners = Lists.newArrayList();
for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
- scanners.add(new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, !isExactAggregation));
+ CubeSegmentScanner scanner;
+ try {
+ scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, !isExactAggregation);
+ } catch (NotEnoughGTInfoException e) {
+ logger.info("Cannot construct Segment {}'s GTInfo, this may due to empty segment or broken metadata");
+ continue;
+ }
+ scanners.add(scanner);
}
if (scanners.isEmpty())
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ddeeb09d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseScan.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseScan.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseScan.java
index 7667830..65a963d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseScan.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseScan.java
@@ -45,7 +45,7 @@ public class HBaseScan {
int colLength = info.getCodeSystem().maxCodeLength(c);
if (rec.get(c).array() != null) {
- Preconditions.checkArgument(colLength == rec.get(c).length(), "ColLength :" + colLength + " not equals cols[c] length: " + rec.get(c).length() + " c is " + c);
+ Preconditions.checkArgument(colLength == rec.get(c).length(), "ColLength :" + colLength + " != cols[c].length: " + rec.get(c).length() + ", c is " + c);
System.arraycopy(rec.get(c).array(), rec.get(c).offset(), buf.array(), buf.offset() + pos, rec.get(c).length());
} else {
Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + pos + colLength, fill);