You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/02/21 13:12:59 UTC
[2/3] kylin git commit: temp
temp
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c1cf570e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c1cf570e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c1cf570e
Branch: refs/heads/mhb0220
Commit: c1cf570e2cf1651bb0e85824bfda155dc82a4fc1
Parents: a7a318c
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Feb 20 17:31:02 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Feb 21 20:55:36 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 6 +-
.../apache/kylin/query/ITFailfastQueryTest.java | 154 +++++++++++++++++++
.../apache/kylin/query/ITKylinQueryTest.java | 18 ---
.../org/apache/kylin/query/KylinTestBase.java | 15 ++
.../apache/kylin/rest/service/QueryService.java | 12 +-
.../coprocessor/endpoint/CubeVisitService.java | 2 +-
6 files changed, 182 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c1cf570e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f7d8452..13d967d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -826,7 +826,7 @@ abstract public class KylinConfigBase implements Serializable {
}
public int getLargeQueryThreshold() {
- return Integer.parseInt(getOptional("kylin.query.large-query-threshold", String.valueOf((int) (getScanThreshold() * 0.1))));
+ return Integer.parseInt(getOptional("kylin.query.large-query-threshold", String.valueOf(1000000)));
}
public int getDerivedInThreshold() {
@@ -865,6 +865,10 @@ abstract public class KylinConfigBase implements Serializable {
return Long.parseLong(this.getOptional("kylin.query.cache-threshold-scan-count", String.valueOf(10 * 1024)));
}
+ public long getQueryScanBytesCacheThreshold() {
+ return Long.parseLong(this.getOptional("kylin.query.cache-threshold-scan-bytes", String.valueOf(1024 * 1024)));
+ }
+
public boolean isQuerySecureEnabled() {
return Boolean.parseBoolean(this.getOptional("kylin.query.security-enabled", "true"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c1cf570e/kylin-it/src/test/java/org/apache/kylin/query/ITFailfastQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITFailfastQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITFailfastQueryTest.java
new file mode 100644
index 0000000..a3720c8
--- /dev/null
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITFailfastQueryTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.query;
+
+import java.io.File;
+import java.util.Map;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.common.exceptions.ResourceLimitExceededException;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.query.routing.Candidate;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import com.google.common.collect.Maps;
+
+public class ITFailfastQueryTest extends KylinTestBase {
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ printInfo("setUp in ITFailfastQueryTest");
+ Map<RealizationType, Integer> priorities = Maps.newHashMap();
+ priorities.put(RealizationType.HYBRID, 0);
+ priorities.put(RealizationType.CUBE, 0);
+ priorities.put(RealizationType.INVERTED_INDEX, 0);
+ Candidate.setPriorities(priorities);
+ joinType = "left";
+ setupAll();
+ }
+
+ @After
+ public void cleanUp() {
+ QueryContext.reset();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ printInfo("tearDown in ITFailfastQueryTest");
+ Candidate.restorePriorities();
+ clean();
+ }
+
+ @Test
+ public void testPartitionExceedMaxScanBytes() throws Exception {
+ String key = "kylin.storage.partition.max-scan-bytes";
+ long saved = KylinConfig.getInstanceFromEnv().getPartitionMaxScanBytes();
+ KylinConfig.getInstanceFromEnv().setProperty(key, "18000");//very low threshold
+
+ boolean meetExpectedException = false;
+ try {
+ String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql/query01.sql";
+ File sqlFile = new File(queryFileName);
+ try {
+ runSQL(sqlFile, false, false);
+ } catch (Exception e) {
+ if (findRoot(e) instanceof ResourceLimitExceededException) {
+ //expected
+ meetExpectedException = true;
+ } else {
+ throw new RuntimeException(e);
+ }
+ }
+
+ if (!meetExpectedException) {
+ throw new RuntimeException("Did not meet expected exception");
+ }
+ } finally {
+ KylinConfig.getInstanceFromEnv().setProperty(key, String.valueOf(saved));
+ }
+ }
+
+ @Test
+ public void testPartitionNotExceedMaxScanBytes() throws Exception {
+ String key = "kylin.storage.partition.max-scan-bytes";
+ long saved = KylinConfig.getInstanceFromEnv().getPartitionMaxScanBytes();
+ KylinConfig.getInstanceFromEnv().setProperty(key, "20000");//enough threshold
+
+ try {
+ String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql/query01.sql";
+ File sqlFile = new File(queryFileName);
+ runSQL(sqlFile, false, false);
+ } finally {
+ KylinConfig.getInstanceFromEnv().setProperty(key, String.valueOf(saved));
+ }
+ }
+
+ @Test
+ public void testQueryExceedMaxScanBytes() throws Exception {
+ String key = "kylin.query.max-scan-bytes";
+ long saved = KylinConfig.getInstanceFromEnv().getQueryMaxScanBytes();
+ KylinConfig.getInstanceFromEnv().setProperty(key, "30000");//very low threshold
+
+ boolean meetExpectedException = false;
+ try {
+ String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql/query01.sql";
+ File sqlFile = new File(queryFileName);
+ try {
+ runSQL(sqlFile, false, false);
+ } catch (Exception e) {
+ if (findRoot(e) instanceof ResourceLimitExceededException) {
+ //expected
+ meetExpectedException = true;
+ } else {
+ throw new RuntimeException(e);
+ }
+ }
+
+ if (!meetExpectedException) {
+ throw new RuntimeException("Did not meet expected exception");
+ }
+ } finally {
+ KylinConfig.getInstanceFromEnv().setProperty(key, String.valueOf(saved));
+ }
+ }
+
+ @Test
+ public void testQueryNotExceedMaxScanBytes() throws Exception {
+ String key = "kylin.query.max-scan-bytes";
+ long saved = KylinConfig.getInstanceFromEnv().getQueryMaxScanBytes();
+ KylinConfig.getInstanceFromEnv().setProperty(key, "40000");//enough threshold
+
+ try {
+ String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql/query01.sql";
+ File sqlFile = new File(queryFileName);
+ runSQL(sqlFile, false, false);
+ } finally {
+ KylinConfig.getInstanceFromEnv().setProperty(key, String.valueOf(saved));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c1cf570e/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 842ce40..1158704 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -74,21 +74,6 @@ public class ITKylinQueryTest extends KylinTestBase {
clean();
}
- protected String getQueryFolderPrefix() {
- return "";
- }
-
- protected Throwable findRoot(Throwable throwable) {
- while (true) {
- if (throwable.getCause() != null) {
- throwable = throwable.getCause();
- } else {
- break;
- }
- }
- return throwable;
- }
-
@Test
public void testTimeoutQuery() throws Exception {
try {
@@ -121,9 +106,6 @@ public class ITKylinQueryTest extends KylinTestBase {
try {
runSQL(sqlFile, false, false);
} catch (SQLException e) {
-
- System.out.println(e.getMessage());
-
if (findRoot(e) instanceof KylinTimeoutException) {
//expected
continue;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c1cf570e/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 402aaa0..fd04b2f 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -692,4 +692,19 @@ public class KylinTestBase {
return OLAPContext.getThreadLocalContexts().iterator().next();
}
+ protected String getQueryFolderPrefix() {
+ return "";
+ }
+
+ protected Throwable findRoot(Throwable throwable) {
+ while (true) {
+ if (throwable.getCause() != null) {
+ throwable = throwable.getCause();
+ } else {
+ break;
+ }
+ }
+ return throwable;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c1cf570e/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 4c02aa4..122b823 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -361,14 +361,16 @@ public class QueryService extends BasicService {
sqlResponse = query(sqlRequest);
long durationThreshold = kylinConfig.getQueryDurationCacheThreshold();
- long scancountThreshold = kylinConfig.getQueryScanCountCacheThreshold();
+ long scanCountThreshold = kylinConfig.getQueryScanCountCacheThreshold();
+ long scanBytesThreshold = kylinConfig.getQueryScanBytesCacheThreshold();
sqlResponse.setDuration(System.currentTimeMillis() - startTime);
logger.info("Stats of SQL response: isException: {}, duration: {}, total scan count {}", //
String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()), String.valueOf(sqlResponse.getTotalScanCount()));
- if (checkCondition(queryCacheEnabled, "query cache is disabled") && //
- checkCondition(!sqlResponse.getIsException(), "query has exception") && //
- checkCondition(sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold, "query is too lightweight with duration: {} ({}), scan count: {} ({})", sqlResponse.getDuration(), durationThreshold, sqlResponse.getTotalScanCount(), scancountThreshold) && //
- checkCondition(sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold(), "query response is too large: {} ({})", sqlResponse.getResults().size(), kylinConfig.getLargeQueryThreshold())) {
+ if (checkCondition(queryCacheEnabled, "query cache is disabled") //
+ && checkCondition(!sqlResponse.getIsException(), "query has exception") //
+ && checkCondition(sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scanCountThreshold || sqlResponse.getTotalScanBytes() > scanBytesThreshold, //
+ "query is too lightweight with duration: {} (threshold {}), scan count: {} (threshold {}), scan bytes: {} (threshold {})", sqlResponse.getDuration(), durationThreshold, sqlResponse.getTotalScanCount(), scanCountThreshold, sqlResponse.getTotalScanBytes(), scanBytesThreshold)
+ && checkCondition(sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold(), "query response is too large: {} ({})", sqlResponse.getResults().size(), kylinConfig.getLargeQueryThreshold())) {
cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest, sqlResponse));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c1cf570e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 61cf067..cde127e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -293,7 +293,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
ResourceTrackingCellListIterator cellListIterator = new ResourceTrackingCellListIterator(
allCellLists,
scanReq.getStorageScanRowNumThreshold(), // for old client (scan threshold)
- request.hasMaxScanBytes() ? Long.MAX_VALUE : request.getMaxScanBytes(), // for new client
+ !request.hasMaxScanBytes() ? Long.MAX_VALUE : request.getMaxScanBytes(), // for new client
scanReq.getTimeout());
IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScans.get(0).hbaseColumns, hbaseColumnsToGT, request.getRowkeyPreambleSize(), behavior.delayToggledOn());