You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/11/13 11:02:13 UTC
[2/2] incubator-kylin git commit: KYLIN-1142 more detailed hitCache
message in query's log
KYLIN-1142 more detailed hitCache message in query's log
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ae0f1a72
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ae0f1a72
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ae0f1a72
Branch: refs/heads/2.x-staging
Commit: ae0f1a72e76f436eab4e0eb4e5d786bde83a719b
Parents: 692438f
Author: honma <ho...@ebay.com>
Authored: Fri Nov 13 17:49:30 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri Nov 13 17:50:30 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/storage/StorageContext.java | 16 +++++++++++-----
.../storage/cache/CacheFledgedDynamicQuery.java | 8 ++++----
.../storage/cache/CacheFledgedStaticQuery.java | 3 ++-
.../kylin/query/enumerator/OLAPEnumerator.java | 1 +
.../kylin/rest/controller/QueryController.java | 2 +-
.../apache/kylin/rest/response/SQLResponse.java | 19 ++++++++++++++-----
.../apache/kylin/rest/service/QueryService.java | 11 +++++++++--
.../kylin/rest/service/QueryServiceTest.java | 2 +-
8 files changed, 43 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ae0f1a72/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index 1643aa4..90f950c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -18,13 +18,11 @@
package org.apache.kylin.storage;
-import java.util.ArrayList;
-import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.realization.SQLDigest;
+
+import com.google.common.collect.Range;
/**
* @author xjiang
@@ -33,7 +31,6 @@ public class StorageContext {
public static final int DEFAULT_THRESHOLD = 1000000;
-
private String connUrl;
private int threshold;
private int limit;
@@ -49,6 +46,8 @@ public class StorageContext {
private Cuboid cuboid;
private boolean partialResultReturned;
+ private Range<Long> reusedPeriod;
+
public StorageContext() {
this.threshold = DEFAULT_THRESHOLD;
this.limit = DEFAULT_THRESHOLD;
@@ -160,4 +159,11 @@ public class StorageContext {
return this.enableCoprocessor;
}
+ public Range<Long> getReusedPeriod() {
+ return reusedPeriod;
+ }
+
+ public void setReusedPeriod(Range<Long> reusedPeriod) {
+ this.reusedPeriod = reusedPeriod;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ae0f1a72/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
index b9ae0e9..b9808ea 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
@@ -62,7 +62,7 @@ public class CacheFledgedDynamicQuery extends AbstractCacheFledgedQuery {
if (enableDynamicCache) {
StreamSQLResult cachedResult = getStreamSQLResult(new StreamSQLDigest(sqlDigest, partitionColRef));
if (cachedResult != null) {
- ret = tryReuseCache(context, sqlDigest, returnTupleInfo, ret, cachedResult);
+ ret = tryReuseCache(context, sqlDigest, returnTupleInfo, cachedResult);
} else {
logger.info("no cache entry for this query");
}
@@ -70,10 +70,8 @@ public class CacheFledgedDynamicQuery extends AbstractCacheFledgedQuery {
if (ret == null) {
ret = underlyingStorage.search(context, sqlDigest, returnTupleInfo);
- noCacheUsed = true;
logger.info("No Cache being used");
} else {
- noCacheUsed = false;
logger.info("Cache being used");
}
@@ -90,7 +88,7 @@ public class CacheFledgedDynamicQuery extends AbstractCacheFledgedQuery {
/**
* if cache is not enough it will try to combine existing cache as well as fresh records
*/
- private ITupleIterator tryReuseCache(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo, ITupleIterator ret, StreamSQLResult cachedResult) {
+ private ITupleIterator tryReuseCache(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo, StreamSQLResult cachedResult) {
Range<Long> reusePeriod = cachedResult.getReusableResults(ts);
logger.info("existing cache: " + cachedResult);
@@ -116,9 +114,11 @@ public class CacheFledgedDynamicQuery extends AbstractCacheFledgedQuery {
});
iTupleIteratorList.add(freshTuples);
+ context.setReusedPeriod(reusePeriod);
return new CompoundTupleIterator(iTupleIteratorList);
} else if (remainings.size() == 0) {
logger.info("The ts range in new query was fully cached");
+ context.setReusedPeriod(reusePeriod);
return new SimpleTupleIterator(cachedResult.reuse(reusePeriod));
} else {
//if using cache causes more than one underlyingStorage searches
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ae0f1a72/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
index 93e5e09..d024cf7 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
@@ -29,10 +29,11 @@ public class CacheFledgedStaticQuery extends AbstractCacheFledgedQuery {
public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo) {
StreamSQLResult cachedResult = getStreamSQLResult(new StreamSQLDigest(sqlDigest, null));
- ITupleIterator ret = null;
+ ITupleIterator ret;
if (cachedResult != null) {
logger.info("using existing cache");
+ context.setReusedPeriod(Ranges.<Long> all());
return new SimpleTupleIterator(cachedResult.reuse(Ranges.<Long> all()));
} else {
logger.info("no existing cache to use");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ae0f1a72/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 07f72b1..aa8b2a7 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -125,6 +125,7 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest, olapContext.returnTupleInfo);
if (logger.isDebugEnabled()) {
logger.debug("return TupleIterator...");
+ logger.debug("Storage cache used for this storage query:" + olapContext.storageContext.getReusedPeriod());
}
return iterator;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ae0f1a72/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index dfd55f1..f60894e 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -227,7 +227,7 @@ public class QueryController extends BasicController {
exceptionCache.get(sqlRequest) != null) {
Element element = exceptionCache.get(sqlRequest);
response = (SQLResponse) element.getObjectValue();
- response.setHitCache(true);
+ response.setHitExceptionCache(true);
}
return response;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ae0f1a72/server/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/response/SQLResponse.java b/server/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
index 6541ef9..f5ef8c5 100644
--- a/server/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
+++ b/server/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
@@ -52,7 +52,9 @@ public class SQLResponse implements Serializable {
private long totalScanCount;
- private boolean hitCache = false;
+ private boolean hitExceptionCache = false;
+
+ private boolean storageCacheUsed = false;
public SQLResponse() {
}
@@ -133,12 +135,19 @@ public class SQLResponse implements Serializable {
this.totalScanCount = totalScanCount;
}
- public boolean isHitCache() {
- return hitCache;
+ public boolean isHitExceptionCache() {
+ return hitExceptionCache;
}
- public void setHitCache(boolean hitCache) {
- this.hitCache = hitCache;
+ public void setHitExceptionCache(boolean hitExceptionCache) {
+ this.hitExceptionCache = hitExceptionCache;
}
+ public boolean isStorageCacheUsed() {
+ return storageCacheUsed;
+ }
+
+ public void setStorageCacheUsed(boolean storageCacheUsed) {
+ this.storageCacheUsed = storageCacheUsed;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ae0f1a72/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 13eb09c..669616b 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -203,8 +203,9 @@ public class QueryService extends BasicService {
final Set<String> realizationNames = new HashSet<String>();
final Set<Long> cuboidIds = new HashSet<Long>();
float duration = response.getDuration() / (float) 1000;
+ boolean storageCacheUsed = false;
- if (!response.isHitCache() && null != OLAPContext.getThreadLocalContexts()) {
+ if (!response.isHitExceptionCache() && null != OLAPContext.getThreadLocalContexts()) {
for (OLAPContext ctx : OLAPContext.getThreadLocalContexts()) {
Cuboid cuboid = ctx.storageContext.getCuboid();
if (cuboid != null) {
@@ -216,6 +217,11 @@ public class QueryService extends BasicService {
String realizationName = ctx.realization.getName();
realizationNames.add(realizationName);
}
+
+ if (ctx.storageContext.getReusedPeriod() != null) {
+ response.setStorageCacheUsed(true);
+ storageCacheUsed = true;
+ }
}
}
@@ -239,7 +245,8 @@ public class QueryService extends BasicService {
stringBuilder.append("Result row count: ").append(resultRowCount).append(newLine);
stringBuilder.append("Accept Partial: ").append(request.isAcceptPartial()).append(newLine);
stringBuilder.append("Is Partial Result: ").append(response.isPartial()).append(newLine);
- stringBuilder.append("Hit Cache: ").append(response.isHitCache()).append(newLine);
+ stringBuilder.append("Hit Exception Cache: ").append(response.isHitExceptionCache()).append(newLine);
+ stringBuilder.append("Storage cache used: ").append(storageCacheUsed).append(newLine);
stringBuilder.append("Message: ").append(response.getExceptionMessage()).append(newLine);
stringBuilder.append("==========================[QUERY]===============================").append(newLine);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ae0f1a72/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
index cc86e1e..119d88b 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
@@ -59,7 +59,7 @@ public class QueryServiceTest extends ServiceTestBase {
request.setSql("select * from test_table");
request.setAcceptPartial(true);
SQLResponse response = new SQLResponse();
- response.setHitCache(true);
+ response.setHitExceptionCache(true);
queryService.logQuery(request, response);
}
}