You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/10/28 08:16:44 UTC
[1/3] incubator-kylin git commit: KYLIN-942 add more backdoor toggles
to help profiling
Repository: incubator-kylin
Updated Branches:
refs/heads/2.x-staging 975733630 -> b12d86783
KYLIN-942 add more backdoor toggles to help profiling
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ef2cc71c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ef2cc71c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ef2cc71c
Branch: refs/heads/2.x-staging
Commit: ef2cc71ca23999bbb9ea28b337319107dfca9e65
Parents: 9757336
Author: honma <ho...@ebay.com>
Authored: Wed Oct 28 15:19:51 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Oct 28 15:19:51 2015 +0800
----------------------------------------------------------------------
.../kylin/common/debug/BackdoorToggles.java | 29 ++++++++++++++++++++
.../kylin/storage/hbase/HBaseStorage.java | 23 ++++++++++++----
.../hbase/cube/v2/CubeSegmentScanner.java | 9 ++++--
3 files changed, 54 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ef2cc71c/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 53e5864..bac7258 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -34,6 +34,14 @@ public class BackdoorToggles {
return getString(DEBUG_TOGGLE_OBSERVER_BEHAVIOR);
}
+ public static String getHbaseCubeQueryVersion() {
+ return getString(DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION);
+ }
+
+ public static String getHbaseCubeQueryProtocol() {
+ return getString(DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL);
+ }
+
public static boolean getDisableFuzzyKey() {
return getBoolean(DEBUG_TOGGLE_DISABLE_FUZZY_KEY);
}
@@ -72,6 +80,27 @@ public class BackdoorToggles {
public final static String DEBUG_TOGGLE_DISABLE_FUZZY_KEY = "DEBUG_TOGGLE_DISABLE_FUZZY_KEY";
/**
+ * set DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION=v1/v2 to control which version CubeStorageQuery to use
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION": "v1"
+ }
+ */
+ public final static String DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION = "DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION";
+
+ /**
+ * set DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL=endpoint/scan to control how to visit hbase cube
+ * this param is only valid when DEBUG_TOGGLE_HBASE_CUBE_QUERY_VERSION set to v2(bdefault)
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL": "scan"
+ }
+ */
+ public final static String DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL = "DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL";
+
+ /**
* set DEBUG_TOGGLE_OBSERVER_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR to control observer behavior for debug/profile usage
*
example:(put it into request body)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ef2cc71c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
index 53465d8..2e2000d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
@@ -19,6 +19,7 @@
package org.apache.kylin.storage.hbase;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.engine.mr.IMROutput;
import org.apache.kylin.engine.mr.IMROutput2;
@@ -39,11 +40,15 @@ import org.apache.kylin.storage.hbase.steps.HBaseMROutput2Transition;
import com.google.common.base.Preconditions;
-@SuppressWarnings("unused")//used by reflection
+@SuppressWarnings("unused")
+//used by reflection
public class HBaseStorage implements IStorage {
private final static boolean allowStorageLayerCache = true;
- private final static String defaultCubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v2.CubeStorageQuery";
+
+ private final static String v2CubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v2.CubeStorageQuery";
+ private final static String v1CubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v1.CubeStorageQuery";
+
private final static String defaultIIStorageQuery = "org.apache.kylin.storage.hbase.ii.InvertedIndexStorageQuery";
@Override
@@ -62,11 +67,19 @@ public class HBaseStorage implements IStorage {
return ret;
}
} else if (realization.getType() == RealizationType.CUBE) {
+
+ String cubeStorageQuery;
+ if ("v1".equalsIgnoreCase(BackdoorToggles.getHbaseCubeQueryVersion())) {
+ cubeStorageQuery = v1CubeStorageQuery;
+ } else {
+ cubeStorageQuery = v2CubeStorageQuery;//by default use v2
+ }
+
ICachableStorageQuery ret;
try {
- ret = (ICachableStorageQuery) Class.forName(defaultCubeStorageQuery).getConstructor(CubeInstance.class).newInstance((CubeInstance) realization);
+ ret = (ICachableStorageQuery) Class.forName(cubeStorageQuery).getConstructor(CubeInstance.class).newInstance((CubeInstance) realization);
} catch (Exception e) {
- throw new RuntimeException("Failed to initialize storage query for " + defaultCubeStorageQuery, e);
+ throw new RuntimeException("Failed to initialize storage query for " + cubeStorageQuery, e);
}
if (allowStorageLayerCache) {
@@ -78,7 +91,7 @@ public class HBaseStorage implements IStorage {
throw new IllegalArgumentException("Unknown realization type " + realization.getType());
}
}
-
+
private static IStorageQuery wrapWithCache(ICachableStorageQuery underlyingStorageEngine, IRealization realization) {
if (underlyingStorageEngine.isDynamic()) {
return new CacheFledgedDynamicQuery(underlyingStorageEngine, getPartitionCol(realization));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ef2cc71c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index 814a359..18ba6fb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -11,6 +11,7 @@ import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.ImmutableBitSet;
@@ -224,8 +225,12 @@ public class CubeSegmentScanner implements IGTScanner {
try {
- CubeHBaseRPC rpc = new CubeHBaseEndpointRPC(cubeSeg, cuboid, info);
- //CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info);
+ CubeHBaseRPC rpc;
+ if ("scan".equalsIgnoreCase(BackdoorToggles.getHbaseCubeQueryProtocol())) {
+ rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info);
+ } else {
+ rpc = new CubeHBaseEndpointRPC(cubeSeg, cuboid, info);//default behavior
+ }
//change previous line to CubeHBaseRPC rpc = new CubeHBaseScanRPC(cubeSeg, cuboid, info);
//to debug locally
[2/3] incubator-kylin git commit: KYLIN-942 make sure only trimmed
gtinfo is pushed
Posted by ma...@apache.org.
KYLIN-942 make sure only trimmed gtinfo is pushed
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4731f583
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4731f583
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4731f583
Branch: refs/heads/2.x-staging
Commit: 4731f583d054aedea7ff58e8cbe99d2538fb06be
Parents: ef2cc71
Author: honma <ho...@ebay.com>
Authored: Wed Oct 28 15:20:23 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Oct 28 15:20:23 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/gridtable/GTScanRange.java | 15 +++++----------
.../storage/hbase/cube/v2/CubeSegmentScanner.java | 2 +-
2 files changed, 6 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4731f583/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
index d5fa6c0..87f51df 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRange.java
@@ -3,9 +3,6 @@ package org.apache.kylin.gridtable;
import java.util.Collections;
import java.util.List;
-import javax.annotation.Nullable;
-
-import com.google.common.base.Function;
import com.google.common.collect.Lists;
public class GTScanRange {
@@ -28,15 +25,13 @@ public class GTScanRange {
}
public GTScanRange replaceGTInfo(final GTInfo gtInfo) {
+ List<GTRecord> newFuzzyKeys = Lists.newArrayList();
+ for (GTRecord input : fuzzyKeys) {
+ newFuzzyKeys.add(new GTRecord(gtInfo, input.maskForEqualHashComp(), input.cols));
+ }
return new GTScanRange(new GTRecord(gtInfo, pkStart.maskForEqualHashComp(), pkStart.cols), //
new GTRecord(gtInfo, pkEnd.maskForEqualHashComp(), pkEnd.cols), //
- Lists.transform(fuzzyKeys, new Function<GTRecord, GTRecord>() {
- @Nullable
- @Override
- public GTRecord apply(GTRecord input) {
- return new GTRecord(gtInfo, input.maskForEqualHashComp(), input.cols);
- }
- }));
+ newFuzzyKeys);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4731f583/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index 18ba6fb..1eed318 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -83,7 +83,7 @@ public class CubeSegmentScanner implements IGTScanner {
GTInfo trimmedInfo = GTInfo.deserialize(trimmedInfoBytes);
for (GTScanRange range : scanRanges) {
- scanRequests.add(new GTScanRequest(trimmedInfo, range,//range.replaceGTInfo(trimmedInfo),
+ scanRequests.add(new GTScanRequest(trimmedInfo, range.replaceGTInfo(trimmedInfo),
gtDimensions, gtAggrGroups, gtAggrMetrics, gtAggrFuncs, gtFilter, allowPreAggregate));
}
[3/3] incubator-kylin git commit: KYLIN-943 enable top n sqls
(reverted from commit e6100b548f67624fc4729ba1afc02f5b2db41a9f)
Posted by ma...@apache.org.
KYLIN-943 enable top n sqls (reverted from commit e6100b548f67624fc4729ba1afc02f5b2db41a9f)
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b12d8678
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b12d8678
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b12d8678
Branch: refs/heads/2.x-staging
Commit: b12d86783221847a5528b2e2d9da20a3a242bdea
Parents: 4731f58
Author: honma <ho...@ebay.com>
Authored: Wed Oct 28 15:21:04 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Oct 28 15:21:04 2015 +0800
----------------------------------------------------------------------
query/src/test/resources/query/sql/query82.sql | 27 ------------------
.../resources/query/sql/query82.sql.disable | 27 ++++++++++++++++++
query/src/test/resources/query/sql/query83.sql | 29 --------------------
.../resources/query/sql/query83.sql.disable | 29 ++++++++++++++++++++
4 files changed, 56 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b12d8678/query/src/test/resources/query/sql/query82.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query82.sql b/query/src/test/resources/query/sql/query82.sql
deleted file mode 100644
index 57e9de0..0000000
--- a/query/src/test/resources/query/sql/query82.sql
+++ /dev/null
@@ -1,27 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select test_kylin_fact.cal_dt, seller_id
- FROM test_kylin_fact
- inner JOIN edw.test_cal_dt as test_cal_dt
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- inner JOIN test_category_groupings
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
- AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- group by
- test_kylin_fact.cal_dt, test_kylin_fact.seller_id order by sum(test_kylin_fact.price) desc limit 20
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b12d8678/query/src/test/resources/query/sql/query82.sql.disable
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query82.sql.disable b/query/src/test/resources/query/sql/query82.sql.disable
new file mode 100644
index 0000000..57e9de0
--- /dev/null
+++ b/query/src/test/resources/query/sql/query82.sql.disable
@@ -0,0 +1,27 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select test_kylin_fact.cal_dt, seller_id
+ FROM test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
+ AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ group by
+ test_kylin_fact.cal_dt, test_kylin_fact.seller_id order by sum(test_kylin_fact.price) desc limit 20
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b12d8678/query/src/test/resources/query/sql/query83.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query83.sql b/query/src/test/resources/query/sql/query83.sql
deleted file mode 100644
index 514beb4..0000000
--- a/query/src/test/resources/query/sql/query83.sql
+++ /dev/null
@@ -1,29 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT
- seller_id
- FROM test_kylin_fact
- inner JOIN edw.test_cal_dt as test_cal_dt
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- inner JOIN test_category_groupings
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
- AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- where test_kylin_fact.cal_dt < DATE '2013-02-01'
- group by
- test_kylin_fact.seller_id order by sum(test_kylin_fact.price) desc limit 20
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b12d8678/query/src/test/resources/query/sql/query83.sql.disable
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query83.sql.disable b/query/src/test/resources/query/sql/query83.sql.disable
new file mode 100644
index 0000000..514beb4
--- /dev/null
+++ b/query/src/test/resources/query/sql/query83.sql.disable
@@ -0,0 +1,29 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+ seller_id
+ FROM test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
+ AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ where test_kylin_fact.cal_dt < DATE '2013-02-01'
+ group by
+ test_kylin_fact.seller_id order by sum(test_kylin_fact.price) desc limit 20