You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/04/13 11:16:14 UTC
kylin git commit: KYLIN-1561 remaing part of
b26b2489baddcfd148eaf8f17330878bbf349048
Repository: kylin
Updated Branches:
refs/heads/master b109948e0 -> 1d4c7bb82
KYLIN-1561 remaing part of b26b2489baddcfd148eaf8f17330878bbf349048
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1d4c7bb8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1d4c7bb8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1d4c7bb8
Branch: refs/heads/master
Commit: 1d4c7bb82059176aa8fb0f2805675f64ad603412
Parents: b109948
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Apr 13 17:16:00 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Apr 13 17:16:06 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 13 +++++++-
.../org/apache/kylin/query/KylinTestBase.java | 4 ++-
.../src/test/resources/query/sql/query99.sql | 33 ++++++++++++++++++++
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 2 +-
.../hbase/cube/v2/CubeSegmentScanner.java | 9 +++++-
5 files changed, 57 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4c7bb8/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f50793e..286c20c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -499,6 +499,18 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(this.getOptional("kylin.query.ignore_unknown_function", "false"));
}
+ public String getQueryStorageVisitPlanner() {
+ return this.getOptional("kylin.query.storage.visit.planner", "org.apache.kylin.gridtable.GTScanRangePlanner");
+ }
+
+ public void setQueryStorageVisitPlanner(String v) {
+ setProperty("kylin.query.storage.visit.planner", v);
+ }
+
+ public int getQueryStorageVisitScanRangeMax() {
+ return Integer.valueOf(this.getOptional("kylin.query.storage.visit.scanrange.max", "1000000"));
+ }
+
public int getHBaseKeyValueSize() {
return Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", "10485760"));
}
@@ -639,5 +651,4 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.cube.mr.engine.v2.class", "org.apache.kylin.engine.mr.MRBatchCubingEngine2");
}
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4c7bb8/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 46aa68d..d3c8f6d 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -84,6 +84,7 @@ public class KylinTestBase {
protected static int compQueryCount = 0;
protected static ArrayList<String> zeroResultQueries = new ArrayList<String>();
+ protected static String ITDirHeader = "";
protected static void closeConnection(Connection connection) {
if (connection != null) {
@@ -108,6 +109,7 @@ public class KylinTestBase {
* @return
*/
protected static List<File> getFilesFromFolder(final File folder, final String fileType) {
+ System.out.println(folder.getAbsolutePath());
Set<File> set = new TreeSet<>(new FileByNameComparator());
for (final File fileEntry : folder.listFiles()) {
if (fileEntry.getName().toLowerCase().endsWith(fileType.toLowerCase())) {
@@ -429,7 +431,7 @@ public class KylinTestBase {
protected int runSQL(File sqlFile, boolean debug, boolean explain) throws Exception {
if (debug) {
System.setProperty("calcite.debug", "true");
- InputStream inputStream = new FileInputStream("src/test/resources/logging.properties");
+ InputStream inputStream = new FileInputStream(ITDirHeader + "src/test/resources/logging.properties");
LogManager.getLogManager().readConfiguration(inputStream);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4c7bb8/kylin-it/src/test/resources/query/sql/query99.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query99.sql b/kylin-it/src/test/resources/query/sql/query99.sql
new file mode 100644
index 0000000..80a537d
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query99.sql
@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+ test_kylin_fact.seller_id
+ ,test_category_groupings.meta_categ_name
+ ,test_kylin_fact.lstg_format_name
+ ,sum(price) as GMV, count(*) as TRANS_CNT
+ FROM test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.lstg_site_id = test_category_groupings.site_id AND test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
+ where test_category_groupings.meta_categ_name='Collectibles'
+ group by test_kylin_fact.seller_id
+ ,test_category_groupings.meta_categ_name
+ ,test_kylin_fact.lstg_format_name
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4c7bb8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 81d5baa..d86b37f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -317,7 +317,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
logger.debug("Serialized scanRequestBytes {} bytes, rawScanBytesString {} bytes", scanRequestByteString.size(), rawScanByteString.size());
- logger.info("The scan {} for segment {} is as below, shard part of start/end key is set to 0", Integer.toHexString(System.identityHashCode(scanRequest)), cubeSeg);
+ logger.info("The scan {} for segment {} is as below with {} separate raw scans, shard part of start/end key is set to 0", Integer.toHexString(System.identityHashCode(scanRequest)), cubeSeg,rawScans.size());
for (RawScan rs : rawScans) {
logScan(rs, cubeSeg.getStorageLocationIdentifier());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4c7bb8/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index 9ed914a..2b55ace 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -23,6 +23,7 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.Set;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.dict.BuildInFunctionTransformer;
@@ -58,7 +59,13 @@ public class CubeSegmentScanner implements IGTScanner {
ITupleFilterTransformer translator = new BuildInFunctionTransformer(cubeSeg.getDimensionEncodingMap());
filter = translator.transform(filter);
- GTScanRangePlanner scanRangePlanner = new GTScanRangePlanner(cubeSeg, cuboid, filter, dimensions, groups, metrics);
+ String plannerName = KylinConfig.getInstanceFromEnv().getQueryStorageVisitPlanner();
+ GTScanRangePlanner scanRangePlanner;
+ try {
+ scanRangePlanner = (GTScanRangePlanner) Class.forName(plannerName).getConstructor(CubeSegment.class, Cuboid.class, TupleFilter.class, Set.class, Set.class, Collection.class).newInstance(cubeSeg, cuboid, filter, dimensions, groups, metrics);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
scanRequest = scanRangePlanner.planScanRequest(allowPreAggregate);
scanner = new ScannerWorker(cubeSeg, cuboid, scanRequest);
}