You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/10/14 14:16:59 UTC
incubator-kylin git commit: minor
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-942 18e7999dc -> ca4552565
minor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ca455256
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ca455256
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ca455256
Branch: refs/heads/KYLIN-942
Commit: ca45525651af4e099bfd48b7634c04abee3f8884
Parents: 18e7999
Author: honma <ho...@ebay.com>
Authored: Wed Oct 14 20:20:45 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Oct 14 20:20:45 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 10 ++------
.../kylin/gridtable/GTScanRangePlanner.java | 26 ++++++++++++++------
.../kylin/gridtable/DictGridTableTest.java | 2 +-
.../kylin/query/test/ITKylinQueryTest.java | 2 +-
query/src/test/resources/query/sql/query85.sql | 26 ++++++++++++++++++++
.../hbase/cube/v1/CubeSegmentTupleIterator.java | 2 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 3 ++-
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 2 +-
.../hbase/cube/v2/CubeSegmentScanner.java | 4 +--
9 files changed, 54 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index fd16434..7452539 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -20,7 +20,6 @@ package org.apache.kylin.cube;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -49,10 +48,9 @@ import com.google.common.collect.Lists;
@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class CubeInstance extends RootPersistentEntity implements IRealization, IBuildable {
- private static final int COST_WEIGHT_DIMENSION = 1;
private static final int COST_WEIGHT_MEASURE = 1;
- private static final int COST_WEIGHT_LOOKUP_TABLE = 1;
- private static final int COST_WEIGHT_INNER_JOIN = 2;
+ private static final int COST_WEIGHT_DIMENSION = 10;
+ private static final int COST_WEIGHT_INNER_JOIN = 100;
public static CubeInstance create(String cubeName, String projectName, CubeDesc cubeDesc) {
CubeInstance cubeInstance = new CubeInstance();
@@ -92,8 +90,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
@JsonProperty("create_time_utc")
private long createTimeUTC;
-
-
private String projectName;
// default constructor for jackson
@@ -358,7 +354,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
for (LookupDesc lookupDesc : this.getDescriptor().getModel().getLookups()) {
// more tables, more cost
- calculatedCost += COST_WEIGHT_LOOKUP_TABLE;
if ("inner".equals(lookupDesc.getJoin().getType())) {
// inner join cost is bigger than left join, as it will filter some records
calculatedCost += COST_WEIGHT_INNER_JOIN;
@@ -447,5 +442,4 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
return getDescriptor().getEngineType();
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index d01fe2c..3068225 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -491,12 +491,17 @@ public class GTScanRangePlanner {
return new ComparatorEx<ByteArray>() {
@Override
public int compare(ByteArray a, ByteArray b) {
- if (a.array() == null)
- return -1;
- else if (b.array() == null)
+ if (a.array() == null) {
+ if (b.array() == null) {
+ return 0;
+ } else {
+ return -1;
+ }
+ } else if (b.array() == null) {
return 1;
- else
+ } else {
return comp.compare(a, b);
+ }
}
};
}
@@ -505,12 +510,17 @@ public class GTScanRangePlanner {
return new ComparatorEx<ByteArray>() {
@Override
public int compare(ByteArray a, ByteArray b) {
- if (a.array() == null)
- return 1;
- else if (b.array() == null)
+ if (a.array() == null) {
+ if (b.array() == null) {
+ return 0;
+ } else {
+ return 1;
+ }
+ } else if (b.array() == null) {
return -1;
- else
+ } else {
return comp.compare(a, b);
+ }
}
};
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index c991e66..283c6e9 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -86,7 +86,7 @@ public class DictGridTableTest {
LogicalTupleFilter filter = and(timeComp1, or(ageComp1, ageComp2));
List<GTScanRange> r = planner.planScanRanges(filter);
assertEquals(1, r.size());
- assertEquals("[1421193600000, 10]-[null, null]", r.get(0).toString());
+ assertEquals("[1421193600000, 10]-[null, 20]", r.get(0).toString());
assertEquals("[[10], [20]]", r.get(0).fuzzyKeys.toString());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
index 72d7c4a..6425f11 100644
--- a/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
@@ -95,7 +95,7 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = "src/test/resources/query/sql/query44.sql";
+ String queryFileName = "src/test/resources/query/sql/query85.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/query/src/test/resources/query/sql/query85.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query85.sql b/query/src/test/resources/query/sql/query85.sql
new file mode 100644
index 0000000..1a51a02
--- /dev/null
+++ b/query/src/test/resources/query/sql/query85.sql
@@ -0,0 +1,26 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select sum(price) as GMV, count(*) as TRANS_CNT FROM test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
+ AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+where test_kylin_fact.cal_dt < DATE '2012-05-01' or test_kylin_fact.cal_dt > DATE '2013-05-01'
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
index 0110fbe..17fac5e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
@@ -183,7 +183,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
info.append(keyRange.getCuboid().getId());
info.append("\nStart: ");
info.append(keyRange.getStartKeyAsString());
- info.append(" - ");
+ info.append(" - ");
info.append(Bytes.toStringBinary(keyRange.getStartKey()));
info.append("\nStop: ");
info.append(keyRange.getStopKeyAsString());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 2e58644..e0f6922 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -140,10 +140,11 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
ExecutorService executorService = Executors.newFixedThreadPool(rawScans.size());
final List<byte[]> rowBlocks = Collections.synchronizedList(Lists.<byte[]> newArrayList());
+ logger.info("Total RawScan range count: " + rawScans.size());
for (RawScan rawScan : rawScans) {
logScan(rawScan, cubeSeg.getStorageLocationIdentifier());
}
-
+
for (final RawScan rawScan : rawScans) {
executorService.submit(new Runnable() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index cc7ec4f..97a7284 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -229,7 +229,7 @@ public abstract class CubeHBaseRPC {
info.append(cuboid.getId());
info.append("\nStart: ");
info.append(rawScan.getStartKeyAsString());
- info.append(" - ");
+ info.append(" - ");
info.append(Bytes.toStringBinary(rawScan.startKey));
info.append("\nStop: ");
info.append(rawScan.getEndKeyAsString());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca455256/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index d49de56..21881a6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -221,8 +221,8 @@ public class CubeSegmentScanner implements IGTScanner {
try {
- //CubeHBaseRPC rpc = new CubeHBaseEndpointRPC(cubeSeg, cuboid, info);
- CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info);
+ CubeHBaseRPC rpc = new CubeHBaseEndpointRPC(cubeSeg, cuboid, info);
+ //CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info);
//change previous line to CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info);
//to debug locally