You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/27 14:43:28 UTC
[1/7] kylin git commit: strenthen storage visit deadline calculation
Repository: kylin
Updated Branches:
refs/heads/master 00de44116 -> 1563eb945
strenthen storage visit deadline calculation
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/53b6c8c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/53b6c8c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/53b6c8c1
Branch: refs/heads/master
Commit: 53b6c8c18033f92feadd2295456fa4f31011b5bb
Parents: 2436140
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Dec 24 17:57:55 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:15:44 2016 +0800
----------------------------------------------------------------------
.../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/53b6c8c1/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 38efecc..de53d0d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -236,7 +236,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
}
final MutableBoolean scanNormalComplete = new MutableBoolean(true);
- final long deadline = scanReq.getTimeout() + this.serviceStartTime;
+ final long deadline = scanReq.getStartTime() + scanReq.getTimeout();
logger.info("deadline is " + deadline);
final long storagePushDownLimit = scanReq.getStoragePushDownLimit();
[4/7] kylin git commit: introduce CheckUtil to make multiple
condition checks clearer in log
Posted by li...@apache.org.
introduce CheckUtil to make multiple condition checks clearer in log
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2436140e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2436140e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2436140e
Branch: refs/heads/master
Commit: 2436140ef093c8d5b061124a5dbc2f5a1c3e9f3f
Parents: 182c565
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Dec 24 16:52:19 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:15:44 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/CheckUtil.java | 33 ++++++++++++++++++++
.../apache/kylin/rest/service/QueryService.java | 17 ++++++----
2 files changed, 44 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2436140e/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
new file mode 100644
index 0000000..ae189f7
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CheckUtil.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.util;
+
+import org.slf4j.LoggerFactory;
+
+public class CheckUtil {
+ public static final org.slf4j.Logger logger = LoggerFactory.getLogger(CheckUtil.class);
+
+ public static boolean checkCondition(boolean condition, String message, Object... args) {
+ if (condition) {
+ return true;
+ } else {
+ logger.debug(message, args);
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2436140e/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 0dd5c5f..bb0342f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.service;
+import static org.apache.kylin.common.util.CheckUtil.checkCondition;
+
import java.io.IOException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
@@ -341,7 +343,9 @@ public class QueryService extends BasicService {
long startTime = System.currentTimeMillis();
SQLResponse sqlResponse = null;
- boolean queryCacheEnabled = kylinConfig.isQueryCacheEnabled() && !BackdoorToggles.getDisableCache();
+ boolean queryCacheEnabled = checkCondition(kylinConfig.isQueryCacheEnabled(), "query cache disabled in KylinConfig") && //
+ checkCondition(!BackdoorToggles.getDisableCache(), "query cache disabled in BackdoorToggles");
+
if (queryCacheEnabled) {
sqlResponse = searchQueryInCache(sqlRequest);
}
@@ -355,12 +359,13 @@ public class QueryService extends BasicService {
sqlResponse.setDuration(System.currentTimeMillis() - startTime);
logger.info("Stats of SQL response: isException: {}, duration: {}, total scan count {}", //
String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()), String.valueOf(sqlResponse.getTotalScanCount()));
- if (queryCacheEnabled && //
- !sqlResponse.getIsException() && //
- (sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold) && //
- (sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold())) { //don't cache too large response
+ if (checkCondition(queryCacheEnabled, "query cache is disabled") && //
+ checkCondition(!sqlResponse.getIsException(), "query has exception") && //
+ checkCondition(sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold, "query is too lightweight with duration: {} ({}), scan count: {} ({})", sqlResponse.getDuration(), durationThreshold, sqlResponse.getTotalScanCount(), scancountThreshold) && //
+ checkCondition(sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold(), "query response is too large: {} ({})", sqlResponse.getResults().size(), kylinConfig.getLargeQueryThreshold())) {
cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest, sqlResponse));
}
+
} else {
sqlResponse.setDuration(System.currentTimeMillis() - startTime);
}
@@ -437,7 +442,7 @@ public class QueryService extends BasicService {
String correctedSql = QueryUtil.massageSql(sqlRequest);
if (!correctedSql.equals(sqlRequest.getSql())) {
logger.info("The corrected query: " + correctedSql);
-
+
//CAUTION: should not change sqlRequest content!
//sqlRequest.setSql(correctedSql);
}
[7/7] kylin git commit: KYLIN-2318 bug fix
Posted by li...@apache.org.
KYLIN-2318 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1563eb94
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1563eb94
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1563eb94
Branch: refs/heads/master
Commit: 1563eb945d0f3c68874934beca18579c96acc100
Parents: bfe8b1a
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Dec 27 12:47:08 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:17:33 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/debug/BackdoorToggles.java | 10 ++++++++++
.../java/org/apache/kylin/rest/service/QueryService.java | 1 +
2 files changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1563eb94/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 8109cf2..95d5d62 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -23,6 +23,8 @@ import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Pair;
+import com.google.common.collect.Maps;
+
/**
* BackdoorToggles and QueryContext are similar because they're both hosting per-query thread local variables.
* The difference is that BackdoorToggles are specified by user input and work for debug purpose. QueryContext
@@ -38,6 +40,14 @@ public class BackdoorToggles {
_backdoorToggles.set(toggles);
}
+ public static void addToggles(Map<String, String> toggles) {
+ Map<String, String> map = _backdoorToggles.get();
+ if (map == null) {
+ setToggles(Maps.<String, String> newHashMap());
+ }
+ _backdoorToggles.get().putAll(toggles);
+ }
+
public static String getCoprocessorBehavior() {
return getString(DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1563eb94/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index bb0342f..de28b17 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -327,6 +327,7 @@ public class QueryService extends BasicService {
}
final String queryId = UUID.randomUUID().toString();
+ BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
QueryContext.setQueryId(queryId);
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
[6/7] kylin git commit: add a toggle for dumpping storage partitions
Posted by li...@apache.org.
add a toggle for dumpping storage partitions
add more log
temp for dumpped partitions
minor fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bfe8b1a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bfe8b1a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bfe8b1a3
Branch: refs/heads/master
Commit: bfe8b1a35c6b2c2f377eeb72295622ecb4d71284
Parents: 596ab51
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Dec 23 13:56:01 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:17:33 2016 +0800
----------------------------------------------------------------------
.../kylin/common/debug/BackdoorToggles.java | 30 ++++++++++++++++++++
.../gtrecord/StorageResponseGTScatter.java | 2 ++
2 files changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bfe8b1a3/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index ca4a19c..8109cf2 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -62,6 +62,14 @@ public class BackdoorToggles {
return getBoolean(DEBUG_TOGGLE_LOCAL_COPROCESSOR);
}
+ public static String getPartitionDumpDir() {
+ return getString(DEBUG_TOGGLE_PARTITION_DUMP_DIR);
+ }
+
+ public static String getDumpedPartitionDir() {
+ return getString(DEBUG_TOGGLE_DUMPED_PARTITION_DIR);
+ }
+
public static int getQueryTimeout() {
String v = getString(DEBUG_TOGGLE_QUERY_TIMEOUT);
if (v == null)
@@ -190,6 +198,28 @@ public class BackdoorToggles {
*/
public final static String DEBUG_TOGGLE_SHARD_ASSIGNMENT = "DEBUG_TOGGLE_SHARD_ASSIGNMENT";
+ /**
+ * set DEBUG_TOGGLE_PARTITION_DUMP_DIR="dir" to dump the partitions from storage.
+ * The dumped partitions are used for performance profiling, for example.
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_PARTITION_DUMP_DIR": "/tmp/dumping"
+ }
+ */
+ public final static String DEBUG_TOGGLE_PARTITION_DUMP_DIR = "DEBUG_TOGGLE_PARTITION_DUMP_DIR";
+
+ /**
+ * set DEBUG_TOGGLE_DUMPED_PARTITION_DIR="dir" to specify the dir to retrieve previously dumped partitions
+ * it's a companion toggle with DEBUG_TOGGLE_PARTITION_DUMP_DIR
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_DUMPED_PARTITION_DIR": "/tmp/dumped"
+ }
+ */
+ public final static String DEBUG_TOGGLE_DUMPED_PARTITION_DIR = "DEBUG_TOGGLE_DUMPED_PARTITION_DIR";
+
// properties on statement may go with this "channel" too
/**
* set ATTR_STATEMENT_MAX_ROWS="maxRows" to statement's max rows property
http://git-wip-us.apache.org/repos/asf/kylin/blob/bfe8b1a3/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index 6283340..dc8746f 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -77,8 +77,10 @@ public class StorageResponseGTScatter implements IGTScanner {
public Iterator<GTRecord> iterator() {
Iterator<Iterator<GTRecord>> shardSubsets = Iterators.transform(blocks, new EndpointResponseGTScatterFunc());
if (storagePushDownLimit != Integer.MAX_VALUE) {
+ logger.info("Using SortedIteratorMergerWithLimit to merge partitions");
return new SortedIteratorMergerWithLimit<GTRecord>(shardSubsets, storagePushDownLimit, GTRecord.getPrimaryKeyComparator()).getIterator();
} else {
+ logger.info("Using Iterators.concat to merge partitions");
return Iterators.concat(shardSubsets);
}
}
[2/7] kylin git commit: KYLIN-2318 query cache is not working
Posted by li...@apache.org.
KYLIN-2318 query cache is not working
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/182c565d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/182c565d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/182c565d
Branch: refs/heads/master
Commit: 182c565df6b2134ef1302c88988ed27b60fefa42
Parents: fa4a5ee
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Dec 24 15:31:13 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:15:44 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/QueryContext.java | 61 ++++++++++++++++++++
.../kylin/common/debug/BackdoorToggles.java | 11 ++--
.../apache/kylin/rest/request/SQLRequest.java | 23 +++++---
.../apache/kylin/rest/service/QueryService.java | 23 ++++----
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 4 +-
5 files changed, 93 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
new file mode 100644
index 0000000..ef0cb14
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+/**
+ * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for comparision
+ */
+public class QueryContext {
+ private static final ThreadLocal<Map<String, String>> _queryContext = new ThreadLocal<Map<String, String>>();
+
+ public final static String KEY_QUERY_ID = "QUERY_ID";
+
+ public static String getQueryId() {
+ return getString(KEY_QUERY_ID);
+ }
+
+ public static void setQueryId(String uuid) {
+ setString(KEY_QUERY_ID, uuid);
+ }
+
+ private static void setString(String key, String value) {
+ Map<String, String> context = _queryContext.get();
+ if (context == null) {
+ Map<String, String> newMap = Maps.newHashMap();
+ newMap.put(key, value);
+ _queryContext.set(newMap);
+ } else {
+ context.put(key, value);
+ }
+ }
+
+ private static String getString(String key) {
+ Map<String, String> context = _queryContext.get();
+ if (context == null) {
+ return null;
+ } else {
+ return context.get(key);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 28f7697..ca4a19c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -24,6 +24,11 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Pair;
/**
+ * BackdoorToggles and QueryContext are similar because they're both hosting per-query thread local variables.
+ * The difference is that BackdoorToggles are specified by user input and work for debug purpose. QueryContext
+ * is used voluntarily by program itself
+ *
+ * BackdoorToggles is part of SQLRequest, QueryContext does not belong to SQLRequest
*/
public class BackdoorToggles {
@@ -65,10 +70,6 @@ public class BackdoorToggles {
return Integer.valueOf(v);
}
- public static String getQueryId() {
- return getString(KEY_QUERY_ID);
- }
-
public static Pair<Short, Short> getShardAssignment() {
String v = getString(DEBUG_TOGGLE_SHARD_ASSIGNMENT);
if (v == null) {
@@ -104,8 +105,6 @@ public class BackdoorToggles {
_backdoorToggles.remove();
}
- public final static String KEY_QUERY_ID = "QUERY_ID";
-
/**
* set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for debug/profile usage
*
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
index bd8b7e2..1896f4f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
@@ -25,6 +25,7 @@ public class SQLRequest implements Serializable {
protected static final long serialVersionUID = 1L;
private String sql;
+
private String project;
private Integer offset = 0;
private Integer limit = 0;
@@ -83,19 +84,25 @@ public class SQLRequest implements Serializable {
this.acceptPartial = acceptPartial;
}
-
@Override
public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
SQLRequest that = (SQLRequest) o;
- if (acceptPartial != that.acceptPartial) return false;
- if (sql != null ? !sql.equals(that.sql) : that.sql != null) return false;
- if (project != null ? !project.equals(that.project) : that.project != null) return false;
- if (offset != null ? !offset.equals(that.offset) : that.offset != null) return false;
- if (limit != null ? !limit.equals(that.limit) : that.limit != null) return false;
+ if (acceptPartial != that.acceptPartial)
+ return false;
+ if (sql != null ? !sql.equals(that.sql) : that.sql != null)
+ return false;
+ if (project != null ? !project.equals(that.project) : that.project != null)
+ return false;
+ if (offset != null ? !offset.equals(that.offset) : that.offset != null)
+ return false;
+ if (limit != null ? !limit.equals(that.limit) : that.limit != null)
+ return false;
return backdoorToggles != null ? backdoorToggles.equals(that.backdoorToggles) : that.backdoorToggles == null;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 82e4a87..0dd5c5f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.DBUtils;
@@ -266,7 +267,7 @@ public class QueryService extends BasicService {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(newLine);
stringBuilder.append("==========================[QUERY]===============================").append(newLine);
- stringBuilder.append("Query Id: ").append(BackdoorToggles.getQueryId()).append(newLine);
+ stringBuilder.append("Query Id: ").append(QueryContext.getQueryId()).append(newLine);
stringBuilder.append("SQL: ").append(request.getSql()).append(newLine);
stringBuilder.append("User: ").append(user).append(newLine);
stringBuilder.append("Success: ").append((null == response.getExceptionMessage())).append(newLine);
@@ -324,13 +325,7 @@ public class QueryService extends BasicService {
}
final String queryId = UUID.randomUUID().toString();
- Map<String, String> toggles = new HashMap<>();
- toggles.put(BackdoorToggles.KEY_QUERY_ID, queryId);
- if (sqlRequest.getBackdoorToggles() != null) {
- toggles.putAll(sqlRequest.getBackdoorToggles());
- }
- sqlRequest.setBackdoorToggles(toggles);
- BackdoorToggles.setToggles(toggles);
+ QueryContext.setQueryId(queryId);
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
String sql = sqlRequest.getSql();
@@ -442,7 +437,9 @@ public class QueryService extends BasicService {
String correctedSql = QueryUtil.massageSql(sqlRequest);
if (!correctedSql.equals(sqlRequest.getSql())) {
logger.info("The corrected query: " + correctedSql);
- sqlRequest.setSql(correctedSql);
+
+ //CAUTION: should not change sqlRequest content!
+ //sqlRequest.setSql(correctedSql);
}
// add extra parameters into olap context, like acceptPartial
@@ -521,12 +518,12 @@ public class QueryService extends BasicService {
}
/**
- * @param sql
+ * @param correctedSql
* @param sqlRequest
* @return
* @throws Exception
*/
- private SQLResponse execute(String sql, SQLRequest sqlRequest) throws Exception {
+ private SQLResponse execute(String correctedSql, SQLRequest sqlRequest) throws Exception {
Connection conn = null;
Statement stat = null;
ResultSet resultSet = null;
@@ -538,7 +535,7 @@ public class QueryService extends BasicService {
conn = cacheService.getOLAPDataSource(sqlRequest.getProject()).getConnection();
if (sqlRequest instanceof PrepareSqlRequest) {
- PreparedStatement preparedState = conn.prepareStatement(sql);
+ PreparedStatement preparedState = conn.prepareStatement(correctedSql);
processStatementAttr(preparedState, sqlRequest);
for (int i = 0; i < ((PrepareSqlRequest) sqlRequest).getParams().length; i++) {
@@ -549,7 +546,7 @@ public class QueryService extends BasicService {
} else {
stat = conn.createStatement();
processStatementAttr(stat, sqlRequest);
- resultSet = stat.executeQuery(sql);
+ resultSet = stat.executeQuery(correctedSql);
}
ResultSetMetaData metaData = resultSet.getMetaData();
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index ebacb26..df1817e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
@@ -158,7 +158,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
builder.setRowkeyPreambleSize(cubeSeg.getRowKeyPreambleSize());
builder.setKylinProperties(kylinConfig.getConfigAsString());
- final String queryId = BackdoorToggles.getQueryId();
+ final String queryId = QueryContext.getQueryId();
if (queryId != null) {
builder.setQueryId(queryId);
}
[5/7] kylin git commit: close OLAPEnumerator at exception
Posted by li...@apache.org.
close OLAPEnumerator at exception
bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/596ab51b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/596ab51b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/596ab51b
Branch: refs/heads/master
Commit: 596ab51b4b98ae0923a134ede56c0ceb2f23bb66
Parents: 53b6c8c
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Dec 24 18:19:02 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:17:30 2016 +0800
----------------------------------------------------------------------
.../kylin/query/enumerator/OLAPEnumerator.java | 33 +++++++++++++-------
1 file changed, 22 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/596ab51b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index c7b3c71..e4fc937 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -61,20 +61,31 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
@Override
public boolean moveNext() {
- if (cursor == null) {
- cursor = queryStorage();
- }
+ try {
+ if (cursor == null) {
+ cursor = queryStorage();
+ }
- if (!cursor.hasNext()) {
- return false;
- }
+ if (!cursor.hasNext()) {
+ return false;
+ }
- ITuple tuple = cursor.next();
- if (tuple == null) {
- return false;
+ ITuple tuple = cursor.next();
+ if (tuple == null) {
+ return false;
+ }
+ convertCurrentRow(tuple);
+ return true;
+ } catch (Exception e) {
+ try {
+ if (cursor != null) {
+ cursor.close();
+ }
+ } catch (Exception ee) {
+ logger.info("Error when closing cursor, ignore it", ee);
+ }
+ throw e;
}
- convertCurrentRow(tuple);
- return true;
}
private Object[] convertCurrentRow(ITuple tuple) {
[3/7] kylin git commit: minor code refactors
Posted by li...@apache.org.
minor code refactors
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fa4a5ee2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fa4a5ee2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fa4a5ee2
Branch: refs/heads/master
Commit: fa4a5ee269eea66eee5b8ab8278305dfe0144c92
Parents: 00de441
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Dec 22 09:59:43 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:15:44 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 6 ++-
.../org/apache/kylin/common/util/BasicTest.java | 1 +
.../gtrecord/DummyPartitionStreamer.java | 40 ++++++++++++++++++++
.../storage/gtrecord/IPartitionStreamer.java | 26 +++++++++++++
.../gtrecord/StorageResponseGTScatter.java | 9 +++--
.../apache/kylin/rest/service/QueryService.java | 6 ++-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 9 +++--
7 files changed, 87 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 8080577..d73b694 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -750,7 +750,7 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true"));
}
- @Deprecated//Limit is good even it's large. This config is meaning less since we already have scan threshold
+ @Deprecated //Limit is good even it's large. This config is meaning less since we already have scan threshold
public int getStoragePushDownLimitMax() {
return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "10000"));
}
@@ -759,6 +759,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.query.scan-threshold", "10000000"));
}
+ public int getLargeQueryThreshold() {
+ return Integer.parseInt(getOptional("kylin.query.large-query-threshold", String.valueOf((int) (getScanThreshold() * 0.1))));
+ }
+
public int getDerivedInThreshold() {
return Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold", "20"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 5eaa011..9105245 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -34,6 +34,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import com.google.common.collect.Iterators;
import org.apache.commons.lang3.time.FastDateFormat;
import org.junit.Ignore;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
new file mode 100644
index 0000000..4caaed0
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/DummyPartitionStreamer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+public class DummyPartitionStreamer implements IPartitionStreamer {
+ private Iterator<byte[]> iterator;
+
+ public DummyPartitionStreamer(Iterator<byte[]> iterator) {
+ this.iterator = iterator;
+ }
+
+ @Override
+ public void close() throws IOException {
+ //do nothing
+ }
+
+ @Override
+ public Iterator<byte[]> asByteArrayIterator() {
+ return this.iterator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IPartitionStreamer.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IPartitionStreamer.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IPartitionStreamer.java
new file mode 100644
index 0000000..42f1151
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IPartitionStreamer.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import java.io.Closeable;
+import java.util.Iterator;
+
+public interface IPartitionStreamer extends Closeable {
+ public Iterator<byte[]> asByteArrayIterator();
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index fe1afd3..6283340 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -42,14 +42,16 @@ public class StorageResponseGTScatter implements IGTScanner {
private static final Logger logger = LoggerFactory.getLogger(StorageResponseGTScatter.class);
private GTInfo info;
+ private IPartitionStreamer partitionStreamer;
private Iterator<byte[]> blocks;
private ImmutableBitSet columns;
private long totalScannedCount;
private int storagePushDownLimit = -1;
- public StorageResponseGTScatter(GTInfo info, Iterator<byte[]> blocks, ImmutableBitSet columns, long totalScannedCount, int storagePushDownLimit) {
+ public StorageResponseGTScatter(GTInfo info, IPartitionStreamer partitionStreamer, ImmutableBitSet columns, long totalScannedCount, int storagePushDownLimit) {
this.info = info;
- this.blocks = blocks;
+ this.partitionStreamer = partitionStreamer;
+ this.blocks = partitionStreamer.asByteArrayIterator();
this.columns = columns;
this.totalScannedCount = totalScannedCount;
this.storagePushDownLimit = storagePushDownLimit;
@@ -67,7 +69,8 @@ public class StorageResponseGTScatter implements IGTScanner {
@Override
public void close() throws IOException {
- //do nothing
+ //If upper consumer failed while consuming the GTRecords, the consumer should call IGTScanner's close method to ensure releasing resource
+ partitionStreamer.close();
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 8810c85..82e4a87 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -360,8 +360,10 @@ public class QueryService extends BasicService {
sqlResponse.setDuration(System.currentTimeMillis() - startTime);
logger.info("Stats of SQL response: isException: {}, duration: {}, total scan count {}", //
String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()), String.valueOf(sqlResponse.getTotalScanCount()));
- if (queryCacheEnabled && !sqlResponse.getIsException() //
- && (sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold)) {
+ if (queryCacheEnabled && //
+ !sqlResponse.getIsException() && //
+ (sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold) && //
+ (sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold())) { //don't cache too large response
cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest, sqlResponse));
}
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa4a5ee2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index d99f80e..ebacb26 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -40,12 +40,13 @@ import org.apache.kylin.common.util.CompressionUtils;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.LoggableCachedThreadPool;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.model.ISegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
import org.apache.kylin.gridtable.IGTScanner;
+import org.apache.kylin.metadata.model.ISegment;
+import org.apache.kylin.storage.gtrecord.DummyPartitionStreamer;
import org.apache.kylin.storage.gtrecord.StorageResponseGTScatter;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
@@ -127,14 +128,14 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
//TODO: raw scan can be constructed at region side to reduce traffic
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
rawScanByteString = serializeRawScans(rawScans);
-
+
int coprocessorTimeout = getCoprocessorTimeoutMillis();
scanRequest.setTimeout(coprocessorTimeout);
scanRequest.clearScanRanges();//since raw scans are sent to coprocessor, we don't need to duplicate sending it
scanRequestByteString = serializeGTScanReq(scanRequest);
final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(shardNum, coprocessorTimeout);
-
+
logger.info("Serialized scanRequestBytes {} bytes, rawScanBytesString {} bytes", scanRequestByteString.size(), rawScanByteString.size());
logger.info("The scan {} for segment {} is as below with {} separate raw scans, shard part of start/end key is set to 0", Integer.toHexString(System.identityHashCode(scanRequest)), cubeSeg, rawScans.size());
@@ -230,7 +231,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
});
}
- return new StorageResponseGTScatter(fullGTInfo, epResultItr, scanRequest.getColumns(), totalScannedCount.get(), scanRequest.getStoragePushDownLimit());
+ return new StorageResponseGTScatter(fullGTInfo, new DummyPartitionStreamer(epResultItr), scanRequest.getColumns(), totalScannedCount.get(), scanRequest.getStoragePushDownLimit());
}
private ByteString serializeGTScanReq(GTScanRequest scanRequest) {