You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/12/30 02:31:01 UTC
[09/42] kylin git commit: KYLIN-2318 query cache is not working
KYLIN-2318 query cache is not working
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/182c565d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/182c565d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/182c565d
Branch: refs/heads/sparkcubing-rebase
Commit: 182c565df6b2134ef1302c88988ed27b60fefa42
Parents: fa4a5ee
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Dec 24 15:31:13 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 27 22:15:44 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/QueryContext.java | 61 ++++++++++++++++++++
.../kylin/common/debug/BackdoorToggles.java | 11 ++--
.../apache/kylin/rest/request/SQLRequest.java | 23 +++++---
.../apache/kylin/rest/service/QueryService.java | 23 ++++----
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 4 +-
5 files changed, 93 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
new file mode 100644
index 0000000..ef0cb14
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+/**
+ * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for comparision
+ */
+public class QueryContext {
+ private static final ThreadLocal<Map<String, String>> _queryContext = new ThreadLocal<Map<String, String>>();
+
+ public final static String KEY_QUERY_ID = "QUERY_ID";
+
+ public static String getQueryId() {
+ return getString(KEY_QUERY_ID);
+ }
+
+ public static void setQueryId(String uuid) {
+ setString(KEY_QUERY_ID, uuid);
+ }
+
+ private static void setString(String key, String value) {
+ Map<String, String> context = _queryContext.get();
+ if (context == null) {
+ Map<String, String> newMap = Maps.newHashMap();
+ newMap.put(key, value);
+ _queryContext.set(newMap);
+ } else {
+ context.put(key, value);
+ }
+ }
+
+ private static String getString(String key) {
+ Map<String, String> context = _queryContext.get();
+ if (context == null) {
+ return null;
+ } else {
+ return context.get(key);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 28f7697..ca4a19c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -24,6 +24,11 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Pair;
/**
+ * BackdoorToggles and QueryContext are similar because they're both hosting per-query thread local variables.
+ * The difference is that BackdoorToggles are specified by user input and work for debug purpose. QueryContext
+ * is used voluntarily by program itself
+ *
+ * BackdoorToggles is part of SQLRequest, QueryContext does not belong to SQLRequest
*/
public class BackdoorToggles {
@@ -65,10 +70,6 @@ public class BackdoorToggles {
return Integer.valueOf(v);
}
- public static String getQueryId() {
- return getString(KEY_QUERY_ID);
- }
-
public static Pair<Short, Short> getShardAssignment() {
String v = getString(DEBUG_TOGGLE_SHARD_ASSIGNMENT);
if (v == null) {
@@ -104,8 +105,6 @@ public class BackdoorToggles {
_backdoorToggles.remove();
}
- public final static String KEY_QUERY_ID = "QUERY_ID";
-
/**
* set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for debug/profile usage
*
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
index bd8b7e2..1896f4f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
@@ -25,6 +25,7 @@ public class SQLRequest implements Serializable {
protected static final long serialVersionUID = 1L;
private String sql;
+
private String project;
private Integer offset = 0;
private Integer limit = 0;
@@ -83,19 +84,25 @@ public class SQLRequest implements Serializable {
this.acceptPartial = acceptPartial;
}
-
@Override
public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
SQLRequest that = (SQLRequest) o;
- if (acceptPartial != that.acceptPartial) return false;
- if (sql != null ? !sql.equals(that.sql) : that.sql != null) return false;
- if (project != null ? !project.equals(that.project) : that.project != null) return false;
- if (offset != null ? !offset.equals(that.offset) : that.offset != null) return false;
- if (limit != null ? !limit.equals(that.limit) : that.limit != null) return false;
+ if (acceptPartial != that.acceptPartial)
+ return false;
+ if (sql != null ? !sql.equals(that.sql) : that.sql != null)
+ return false;
+ if (project != null ? !project.equals(that.project) : that.project != null)
+ return false;
+ if (offset != null ? !offset.equals(that.offset) : that.offset != null)
+ return false;
+ if (limit != null ? !limit.equals(that.limit) : that.limit != null)
+ return false;
return backdoorToggles != null ? backdoorToggles.equals(that.backdoorToggles) : that.backdoorToggles == null;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 82e4a87..0dd5c5f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.DBUtils;
@@ -266,7 +267,7 @@ public class QueryService extends BasicService {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(newLine);
stringBuilder.append("==========================[QUERY]===============================").append(newLine);
- stringBuilder.append("Query Id: ").append(BackdoorToggles.getQueryId()).append(newLine);
+ stringBuilder.append("Query Id: ").append(QueryContext.getQueryId()).append(newLine);
stringBuilder.append("SQL: ").append(request.getSql()).append(newLine);
stringBuilder.append("User: ").append(user).append(newLine);
stringBuilder.append("Success: ").append((null == response.getExceptionMessage())).append(newLine);
@@ -324,13 +325,7 @@ public class QueryService extends BasicService {
}
final String queryId = UUID.randomUUID().toString();
- Map<String, String> toggles = new HashMap<>();
- toggles.put(BackdoorToggles.KEY_QUERY_ID, queryId);
- if (sqlRequest.getBackdoorToggles() != null) {
- toggles.putAll(sqlRequest.getBackdoorToggles());
- }
- sqlRequest.setBackdoorToggles(toggles);
- BackdoorToggles.setToggles(toggles);
+ QueryContext.setQueryId(queryId);
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
String sql = sqlRequest.getSql();
@@ -442,7 +437,9 @@ public class QueryService extends BasicService {
String correctedSql = QueryUtil.massageSql(sqlRequest);
if (!correctedSql.equals(sqlRequest.getSql())) {
logger.info("The corrected query: " + correctedSql);
- sqlRequest.setSql(correctedSql);
+
+ //CAUTION: should not change sqlRequest content!
+ //sqlRequest.setSql(correctedSql);
}
// add extra parameters into olap context, like acceptPartial
@@ -521,12 +518,12 @@ public class QueryService extends BasicService {
}
/**
- * @param sql
+ * @param correctedSql
* @param sqlRequest
* @return
* @throws Exception
*/
- private SQLResponse execute(String sql, SQLRequest sqlRequest) throws Exception {
+ private SQLResponse execute(String correctedSql, SQLRequest sqlRequest) throws Exception {
Connection conn = null;
Statement stat = null;
ResultSet resultSet = null;
@@ -538,7 +535,7 @@ public class QueryService extends BasicService {
conn = cacheService.getOLAPDataSource(sqlRequest.getProject()).getConnection();
if (sqlRequest instanceof PrepareSqlRequest) {
- PreparedStatement preparedState = conn.prepareStatement(sql);
+ PreparedStatement preparedState = conn.prepareStatement(correctedSql);
processStatementAttr(preparedState, sqlRequest);
for (int i = 0; i < ((PrepareSqlRequest) sqlRequest).getParams().length; i++) {
@@ -549,7 +546,7 @@ public class QueryService extends BasicService {
} else {
stat = conn.createStatement();
processStatementAttr(stat, sqlRequest);
- resultSet = stat.executeQuery(sql);
+ resultSet = stat.executeQuery(correctedSql);
}
ResultSetMetaData metaData = resultSet.getMetaData();
http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index ebacb26..df1817e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
@@ -158,7 +158,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
builder.setRowkeyPreambleSize(cubeSeg.getRowKeyPreambleSize());
builder.setKylinProperties(kylinConfig.getConfigAsString());
- final String queryId = BackdoorToggles.getQueryId();
+ final String queryId = QueryContext.getQueryId();
if (queryId != null) {
builder.setQueryId(queryId);
}