You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/10/29 11:27:53 UTC
[kylin] 01/12: KYLIN-2894 Query cache expiration strategy switches
from manual invalidation to signature checking
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f12088637347385ba1d81eb345c39d2276c2bbe3
Author: Wang Ken <mi...@ebay.com>
AuthorDate: Mon Sep 25 13:59:46 2017 +0800
KYLIN-2894 Query cache expiration strategy switches from manual invalidation to signature checking
---
.../org/apache/kylin/common/KylinConfigBase.java | 5 +
.../apache/kylin/rest/response/SQLResponse.java | 14 +-
.../apache/kylin/rest/service/CacheService.java | 4 +-
.../apache/kylin/rest/service/QueryService.java | 100 ++++++++-----
.../kylin/rest/signature/ComponentSignature.java | 31 ++++
.../rest/signature/RealizationSetCalculator.java | 101 +++++++++++++
.../kylin/rest/signature/RealizationSignature.java | 164 +++++++++++++++++++++
.../kylin/rest/signature/SegmentSignature.java | 65 ++++++++
.../kylin/rest/signature/SignatureCalculator.java | 28 ++++
.../kylin/rest/util/SQLResponseSignatureUtil.java | 68 +++++++++
.../java/org/apache/kylin/rest/bean/BeanTest.java | 2 +-
.../kylin/rest/controller/QueryControllerTest.java | 2 +-
12 files changed, 545 insertions(+), 39 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5577307..135d6e6 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1553,6 +1553,11 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.query.realization-filter", null);
}
+ public String getSQLResponseSignatureClass() {
+ return this.getOptional("kylin.query.signature-class",
+ "org.apache.kylin.rest.signature.RealizationSetCalculator");
+ }
+
// ============================================================================
// SERVER
// ============================================================================
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
index 0bdf037..0502798 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
@@ -76,6 +76,9 @@ public class SQLResponse implements Serializable {
protected String traceUrl = null;
+ // it's sql response signature for cache checking, no need to return and should be JsonIgnore
+ protected String signature;
+
public SQLResponse() {
}
@@ -205,7 +208,16 @@ public class SQLResponse implements Serializable {
public void setTraceUrl(String traceUrl) {
this.traceUrl = traceUrl;
}
-
+
+ @JsonIgnore
+ public String getSignature() {
+ return signature;
+ }
+
+ public void setSignature(String signature) {
+ this.signature = signature;
+ }
+
@JsonIgnore
public List<QueryContext.CubeSegmentStatisticsResult> getCubeSegmentStatisticsList() {
try {
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 10ab90b..67d49d9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -117,8 +117,8 @@ public class CacheService extends BasicService implements InitializingBean {
public void cleanDataCache(String project) {
if (cacheManager != null) {
- logger.info("cleaning cache for project " + project + " (currently remove all entries)");
- cacheManager.getCache(QueryService.SUCCESS_QUERY_CACHE).removeAll();
+ logger.info("cleaning cache for project " + project + " (currently remove exception entries)");
+ // cacheManager.getCache(QueryService.SUCCESS_QUERY_CACHE).removeAll();
cacheManager.getCache(QueryService.EXCEPTION_QUERY_CACHE).removeAll();
} else {
logger.warn("skip cleaning cache for project " + project);
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 8262472..fb13ff5 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -110,6 +110,7 @@ import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.AclPermissionUtil;
import org.apache.kylin.rest.util.QueryRequestLimits;
+import org.apache.kylin.rest.util.SQLResponseSignatureUtil;
import org.apache.kylin.rest.util.TableauInterceptor;
import org.apache.kylin.storage.hybrid.HybridInstance;
import org.apache.kylin.storage.hybrid.HybridManager;
@@ -117,18 +118,17 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.cache.Cache;
+import org.springframework.cache.CacheManager;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import com.google.common.collect.Lists;
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.CacheManager;
-import net.sf.ehcache.Element;
-
/**
* @author xduo
*/
@@ -226,7 +226,7 @@ public class QueryService extends BasicService {
columnMetas.add(new SelectedColumnMeta(false, false, false, false, 1, false, Integer.MAX_VALUE, "c0", "c0",
null, null, null, Integer.MAX_VALUE, 128, 1, "char", false, false, false));
- return buildSqlResponse(true, r.getFirst(), columnMetas);
+ return buildSqlResponse(sqlRequest.getProject(), true, r.getFirst(), columnMetas);
} catch (Exception e) {
logger.info("pushdown engine failed to finish current non-select query");
@@ -313,6 +313,12 @@ public class QueryService extends BasicService {
}
}
+ if (realizationNames.isEmpty()) {
+ if (!Strings.isNullOrEmpty(response.getCube())) {
+ realizationNames.addAll(Lists.newArrayList(response.getCube().split(",")));
+ }
+ }
+
int resultRowCount = 0;
if (!response.getIsException() && response.getResults() != null) {
resultRowCount = response.getResults().size();
@@ -458,6 +464,8 @@ public class QueryService extends BasicService {
String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()),
String.valueOf(sqlResponse.getTotalScanCount()));
if (checkCondition(queryCacheEnabled, "query cache is disabled") //
+ && checkCondition(!Strings.isNullOrEmpty(sqlResponse.getCube()),
+ "query does not hit cube nor hybrid") //
&& checkCondition(!sqlResponse.getIsException(), "query has exception") //
&& checkCondition(
!(sqlResponse.isPushDown()
@@ -473,7 +481,7 @@ public class QueryService extends BasicService {
&& checkCondition(sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold(),
"query response is too large: {} ({})", sqlResponse.getResults().size(),
kylinConfig.getLargeQueryThreshold())) {
- cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest.getCacheKey(), sqlResponse));
+ cacheManager.getCache(SUCCESS_QUERY_CACHE).put(sqlRequest.getCacheKey(), sqlResponse);
}
} catch (Throwable e) { // calcite may throw AssertError
@@ -482,15 +490,13 @@ public class QueryService extends BasicService {
logger.error("Exception while executing query", e);
String errMsg = makeErrorMsgUserFriendly(e);
- sqlResponse = new SQLResponse(null, null, null, 0, true, errMsg, false, false);
- sqlResponse.setTotalScanCount(queryContext.getScannedRows());
- sqlResponse.setTotalScanBytes(queryContext.getScannedBytes());
- sqlResponse.setCubeSegmentStatisticsList(queryContext.getCubeSegmentStatisticsResultList());
+ sqlResponse = buildSqlResponse(sqlRequest.getProject(), false, null, null, true, errMsg);
+ sqlResponse.setThrowable(e.getCause() == null ? e : ExceptionUtils.getRootCause(e));
if (queryCacheEnabled && e.getCause() != null
&& ExceptionUtils.getRootCause(e) instanceof ResourceLimitExceededException) {
Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
- exceptionCache.put(new Element(sqlRequest.getCacheKey(), sqlResponse));
+ exceptionCache.put(sqlRequest.getCacheKey(), sqlResponse);
}
}
return sqlResponse;
@@ -515,22 +521,36 @@ public class QueryService extends BasicService {
}
public SQLResponse searchQueryInCache(SQLRequest sqlRequest) {
- SQLResponse response = null;
- Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
- Cache successCache = cacheManager.getCache(SUCCESS_QUERY_CACHE);
-
- Element element = null;
- if ((element = exceptionCache.get(sqlRequest.getCacheKey())) != null) {
- logger.info("The sqlResponse is found in EXCEPTION_QUERY_CACHE");
- response = (SQLResponse) element.getObjectValue();
- response.setHitExceptionCache(true);
- } else if ((element = successCache.get(sqlRequest.getCacheKey())) != null) {
- logger.info("The sqlResponse is found in SUCCESS_QUERY_CACHE");
- response = (SQLResponse) element.getObjectValue();
- response.setStorageCacheUsed(true);
+ String[] cacheTypes = new String[] { EXCEPTION_QUERY_CACHE, SUCCESS_QUERY_CACHE };
+ for (String cacheType : cacheTypes) {
+ Cache cache = cacheManager.getCache(cacheType);
+ Cache.ValueWrapper wrapper = cache.get(sqlRequest.getCacheKey());
+ if (wrapper == null) {
+ continue;
+ }
+ SQLResponse response = (SQLResponse) wrapper.get();
+ if (response == null) {
+ return null;
+ }
+ logger.info("The sqlResponse is found in " + cacheType);
+ if (!SQLResponseSignatureUtil.checkSignature(getConfig(), response, sqlRequest.getProject())) {
+ logger.info("The sql response signature is changed. Remove it from QUERY_CACHE.");
+ cache.evict(sqlRequest.getCacheKey());
+ return null;
+ } else {
+ switch (cacheType) {
+ case EXCEPTION_QUERY_CACHE:
+ response.setHitExceptionCache(true);
+ break;
+ case SUCCESS_QUERY_CACHE:
+ response.setStorageCacheUsed(true);
+ break;
+ default:
+ }
+ }
+ return response;
}
-
- return response;
+ return null;
}
private SQLResponse queryWithSqlMassage(SQLRequest sqlRequest) throws Exception {
@@ -579,7 +599,8 @@ public class QueryService extends BasicService {
List<List<String>> results = Lists.newArrayList();
List<SelectedColumnMeta> columnMetas = Lists.newArrayList();
if (BackdoorToggles.getPrepareOnly()) {
- return getPrepareOnlySqlResponse(correctedSql, conn, false, results, columnMetas);
+ return getPrepareOnlySqlResponse(sqlRequest.getProject(), correctedSql, conn, false, results,
+ columnMetas);
}
if (!isPrepareRequest) {
return executeRequest(correctedSql, sqlRequest, conn);
@@ -893,7 +914,7 @@ public class QueryService extends BasicService {
close(resultSet, stat, null); //conn is passed in, not my duty to close
}
- return buildSqlResponse(isPushDown, r.getFirst(), r.getSecond());
+ return buildSqlResponse(sqlRequest.getProject(), isPushDown, r.getFirst(), r.getSecond());
}
private SQLResponse executePrepareRequest(String correctedSql, PrepareSqlRequest sqlRequest,
@@ -921,7 +942,7 @@ public class QueryService extends BasicService {
DBUtils.closeQuietly(resultSet);
}
- return buildSqlResponse(isPushDown, r.getFirst(), r.getSecond());
+ return buildSqlResponse(sqlRequest.getProject(), isPushDown, r.getFirst(), r.getSecond());
}
private Pair<List<List<String>>, List<SelectedColumnMeta>> pushDownQuery(SQLRequest sqlRequest, String correctedSql,
@@ -972,7 +993,8 @@ public class QueryService extends BasicService {
return QueryUtil.makeErrorMsgUserFriendly(e);
}
- private SQLResponse getPrepareOnlySqlResponse(String correctedSql, Connection conn, Boolean isPushDown,
+ private SQLResponse getPrepareOnlySqlResponse(String projectName, String correctedSql, Connection conn,
+ Boolean isPushDown,
List<List<String>> results, List<SelectedColumnMeta> columnMetas) throws SQLException {
CalcitePrepareImpl.KYLIN_ONLY_PREPARE.set(true);
@@ -1018,7 +1040,7 @@ public class QueryService extends BasicService {
DBUtils.closeQuietly(preparedStatement);
}
- return buildSqlResponse(isPushDown, results, columnMetas);
+ return buildSqlResponse(projectName, isPushDown, results, columnMetas);
}
private boolean isPrepareStatementWithParams(SQLRequest sqlRequest) {
@@ -1028,10 +1050,17 @@ public class QueryService extends BasicService {
return false;
}
- private SQLResponse buildSqlResponse(Boolean isPushDown, List<List<String>> results,
+ private SQLResponse buildSqlResponse(String projectName, Boolean isPushDown, List<List<String>> results,
List<SelectedColumnMeta> columnMetas) {
+ return buildSqlResponse(projectName, isPushDown, results, columnMetas, false, null);
+ }
+
+ private SQLResponse buildSqlResponse(String projectName, Boolean isPushDown, List<List<String>> results,
+ List<SelectedColumnMeta> columnMetas, boolean isException, String exceptionMessage) {
boolean isPartialResult = false;
+
+ List<String> realizations = Lists.newLinkedList();
StringBuilder cubeSb = new StringBuilder();
StringBuilder logSb = new StringBuilder("Processed rows for each storageContext: ");
QueryContext queryContext = QueryContextFacade.current();
@@ -1049,17 +1078,20 @@ public class QueryService extends BasicService {
realizationName = ctx.realization.getName();
realizationType = ctx.realization.getStorageType();
+
+ realizations.add(realizationName);
}
queryContext.setContextRealization(ctx.id, realizationName, realizationType);
}
}
logger.info(logSb.toString());
- SQLResponse response = new SQLResponse(columnMetas, results, cubeSb.toString(), 0, false, null, isPartialResult,
- isPushDown);
+ SQLResponse response = new SQLResponse(columnMetas, results, cubeSb.toString(), 0, isException,
+ exceptionMessage, isPartialResult, isPushDown);
response.setTotalScanCount(queryContext.getScannedRows());
response.setTotalScanBytes(queryContext.getScannedBytes());
response.setCubeSegmentStatisticsList(queryContext.getCubeSegmentStatisticsResultList());
+ response.setSignature(SQLResponseSignatureUtil.createSignature(getConfig(), response, projectName));
return response;
}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/signature/ComponentSignature.java b/server-base/src/main/java/org/apache/kylin/rest/signature/ComponentSignature.java
new file mode 100644
index 0000000..7b3a2e4
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/signature/ComponentSignature.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.signature;
+
+import java.io.Serializable;
+
+abstract class ComponentSignature<T extends ComponentSignature> implements Serializable, Comparable<T> {
+
+ public abstract String getKey();
+
+ @Override
+ public int compareTo(T o) {
+ return getKey().compareTo(o.getKey());
+ }
+}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/signature/RealizationSetCalculator.java b/server-base/src/main/java/org/apache/kylin/rest/signature/RealizationSetCalculator.java
new file mode 100644
index 0000000..63139f3
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/signature/RealizationSetCalculator.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.signature;
+
+import java.security.MessageDigest;
+import java.util.Set;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.rest.response.SQLResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.CharMatcher;
+import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+
+public class RealizationSetCalculator implements SignatureCalculator {
+
+ public static final Logger logger = LoggerFactory.getLogger(RealizationSetCalculator.class);
+
+ @Override
+ public String calculateSignature(KylinConfig config, SQLResponse sqlResponse, ProjectInstance project) {
+ Set<String> realizations = getRealizations(config, sqlResponse.getCube(), project);
+ if (realizations == null) {
+ return null;
+ }
+ Set<RealizationSignature> signatureSet = Sets.newTreeSet();
+ for (String realization : realizations) {
+ RealizationSignature realizationSignature = getRealizationSignature(config, realization);
+ if (realizationSignature != null) {
+ signatureSet.add(realizationSignature);
+ }
+ }
+ if (signatureSet.isEmpty()) {
+ return null;
+ }
+ try {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ byte[] signature = md.digest(signatureSet.toString().getBytes("UTF-8"));
+ return new String(Base64.encodeBase64(signature), "UTF-8");
+ } catch (Exception e) {
+ logger.warn("Failed to calculate signature due to " + e);
+ return null;
+ }
+ }
+
+ protected Set<String> getRealizations(KylinConfig config, String cubes, ProjectInstance project) {
+ if (Strings.isNullOrEmpty(cubes)) {
+ return null;
+ }
+ String[] realizations = parseNamesFromCanonicalNames(cubes.split(","));
+ return Sets.newHashSet(realizations);
+ }
+
+ protected static RealizationSignature getRealizationSignature(KylinConfig config, String realizationName) {
+ RealizationSignature result = RealizationSignature.HybridSignature.getHybridSignature(config, realizationName);
+ if (result == null) {
+ result = RealizationSignature.CubeSignature.getCubeSignature(config, realizationName);
+ }
+ return result;
+ }
+
+ private static String[] parseNamesFromCanonicalNames(String[] canonicalNames) {
+ String[] result = new String[canonicalNames.length];
+ for (int i = 0; i < canonicalNames.length; i++) {
+ result[i] = parseCanonicalName(canonicalNames[i]).getSecond();
+ }
+ return result;
+ }
+
+ /**
+ * @param canonicalName
+ * @return type and name pair for realization
+ */
+ private static Pair<String, String> parseCanonicalName(String canonicalName) {
+ Iterable<String> parts = Splitter.on(CharMatcher.anyOf("[]=,")).split(canonicalName);
+ String[] partsStr = Iterables.toArray(parts, String.class);
+ return new Pair<>(partsStr[0], partsStr[2]);
+ }
+}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/signature/RealizationSignature.java b/server-base/src/main/java/org/apache/kylin/rest/signature/RealizationSignature.java
new file mode 100644
index 0000000..9e54085
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/signature/RealizationSignature.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.signature;
+
+import java.util.List;
+import java.util.Set;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.storage.hybrid.HybridInstance;
+import org.apache.kylin.storage.hybrid.HybridManager;
+
+import com.google.common.collect.Sets;
+
+public abstract class RealizationSignature extends ComponentSignature<RealizationSignature> {
+
+ static class CubeSignature extends RealizationSignature {
+ public final String name;
+ public final RealizationStatusEnum status;
+ public final Set<SegmentSignature> segmentSignatureSet;
+
+ private CubeSignature(String name, RealizationStatusEnum status, Set<SegmentSignature> segmentSignatureSet) {
+ this.name = name;
+ this.status = status;
+ this.segmentSignatureSet = segmentSignatureSet;
+ }
+
+ public String getKey() {
+ return name;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ CubeSignature that = (CubeSignature) o;
+
+ if (name != null ? !name.equals(that.name) : that.name != null)
+ return false;
+ if (status != that.status)
+ return false;
+ return segmentSignatureSet != null ? segmentSignatureSet.equals(that.segmentSignatureSet)
+ : that.segmentSignatureSet == null;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = name != null ? name.hashCode() : 0;
+ result = 31 * result + (status != null ? status.hashCode() : 0);
+ result = 31 * result + (segmentSignatureSet != null ? segmentSignatureSet.hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return name + "-" + status + ":"
+ + (segmentSignatureSet != null ? Sets.newTreeSet(segmentSignatureSet) : null);
+ }
+
+ static CubeSignature getCubeSignature(KylinConfig config, String realizationName) {
+ CubeInstance cubeInstance = CubeManager.getInstance(config).getCube(realizationName);
+ if (cubeInstance == null) {
+ return null;
+ }
+ if (!cubeInstance.isReady()) {
+ return new CubeSignature(realizationName, RealizationStatusEnum.DISABLED, null);
+ }
+ List<CubeSegment> readySegments = cubeInstance.getSegments(SegmentStatusEnum.READY);
+ Set<SegmentSignature> segmentSignatureSet = Sets.newHashSetWithExpectedSize(readySegments.size());
+ for (CubeSegment cubeSeg : readySegments) {
+ segmentSignatureSet.add(new SegmentSignature(cubeSeg.getName(), cubeSeg.getLastBuildTime()));
+ }
+ return new CubeSignature(realizationName, RealizationStatusEnum.READY, segmentSignatureSet);
+ }
+ }
+
+ static class HybridSignature extends RealizationSignature {
+ public final String name;
+ public final Set<RealizationSignature> realizationSignatureSet;
+
+ private HybridSignature(String name, Set<RealizationSignature> realizationSignatureSet) {
+ this.name = name;
+ this.realizationSignatureSet = realizationSignatureSet;
+ }
+
+ public String getKey() {
+ return name;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ HybridSignature that = (HybridSignature) o;
+
+ if (name != null ? !name.equals(that.name) : that.name != null)
+ return false;
+ return realizationSignatureSet != null ? realizationSignatureSet.equals(that.realizationSignatureSet)
+ : that.realizationSignatureSet == null;
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = name != null ? name.hashCode() : 0;
+ result = 31 * result + (realizationSignatureSet != null ? realizationSignatureSet.hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return name + ":" + (realizationSignatureSet != null ? Sets.newTreeSet(realizationSignatureSet) : null);
+ }
+
+ static HybridSignature getHybridSignature(KylinConfig config, String realizationName) {
+ HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(realizationName);
+ if (hybridInstance == null) {
+ return null;
+ }
+ IRealization[] realizations = hybridInstance.getRealizations();
+ Set<RealizationSignature> realizationSignatureSet = Sets.newHashSetWithExpectedSize(realizations.length);
+ for (IRealization realization : realizations) {
+ RealizationSignature realizationSignature = null;
+ if (realization.getType() == RealizationType.CUBE) {
+ realizationSignature = CubeSignature.getCubeSignature(config, realization.getName());
+ } else if (realization.getType() == RealizationType.HYBRID) {
+ realizationSignature = getHybridSignature(config, realization.getName());
+ }
+ if (realizationSignature != null) {
+ realizationSignatureSet.add(realizationSignature);
+ }
+ }
+ return new HybridSignature(realizationName, realizationSignatureSet);
+ }
+ }
+}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/signature/SegmentSignature.java b/server-base/src/main/java/org/apache/kylin/rest/signature/SegmentSignature.java
new file mode 100644
index 0000000..800dd99
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/signature/SegmentSignature.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.signature;
+
+class SegmentSignature extends ComponentSignature<SegmentSignature> {
+ public final String name;
+ public final long lastBuildTime;
+
+ public SegmentSignature(String name, long lastBuildTime) {
+ this.name = name;
+ this.lastBuildTime = lastBuildTime;
+ }
+
+ public String getKey() {
+ return name;
+ }
+
+ @Override
+ public int compareTo(SegmentSignature o) {
+ return name.compareTo(o.name);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ SegmentSignature that = (SegmentSignature) o;
+
+ if (lastBuildTime != that.lastBuildTime)
+ return false;
+ return name != null ? name.equals(that.name) : that.name == null;
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = name != null ? name.hashCode() : 0;
+ result = 31 * result + (int) (lastBuildTime ^ (lastBuildTime >>> 32));
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return name + ":" + lastBuildTime;
+ }
+}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/signature/SignatureCalculator.java b/server-base/src/main/java/org/apache/kylin/rest/signature/SignatureCalculator.java
new file mode 100644
index 0000000..1f94beb
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/signature/SignatureCalculator.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.signature;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.rest.response.SQLResponse;
+
+public interface SignatureCalculator {
+
+ String calculateSignature(KylinConfig config, SQLResponse sqlResponse, ProjectInstance project);
+}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/SQLResponseSignatureUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/SQLResponseSignatureUtil.java
new file mode 100644
index 0000000..c6d3507
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/SQLResponseSignatureUtil.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.util;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.rest.response.SQLResponse;
+import org.apache.kylin.rest.signature.RealizationSetCalculator;
+import org.apache.kylin.rest.signature.SignatureCalculator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+public class SQLResponseSignatureUtil {
+
+ public static final Logger logger = LoggerFactory.getLogger(SQLResponseSignatureUtil.class);
+
+ public static boolean checkSignature(KylinConfig config, SQLResponse sqlResponse, String projectName) {
+ String old = sqlResponse.getSignature();
+ if (old == null) {
+ return false;
+ }
+ String latest = createSignature(config, sqlResponse, projectName);
+ return old.equals(latest);
+ }
+
+ public static String createSignature(KylinConfig config, SQLResponse sqlResponse, String projectName) {
+ ProjectInstance project = ProjectManager.getInstance(config).getProject(projectName);
+ Preconditions.checkNotNull(project);
+
+ SignatureCalculator signatureCalculator;
+ try {
+ Class signatureClass = getSignatureClass(project.getConfig());
+ signatureCalculator = (SignatureCalculator) signatureClass.getConstructor().newInstance();
+ } catch (Exception e) {
+ logger.warn("Will use default signature since fail to construct signature due to " + e);
+ signatureCalculator = new RealizationSetCalculator();
+ }
+ return signatureCalculator.calculateSignature(config, sqlResponse, project);
+ }
+
+ private static Class getSignatureClass(KylinConfig config) {
+ try {
+ return Class.forName(config.getSQLResponseSignatureClass());
+ } catch (ClassNotFoundException e) {
+ logger.warn("Will use default signature since cannot find class " + config.getSQLResponseSignatureClass());
+ return RealizationSetCalculator.class;
+ }
+ }
+}
diff --git a/server-base/src/test/java/org/apache/kylin/rest/bean/BeanTest.java b/server-base/src/test/java/org/apache/kylin/rest/bean/BeanTest.java
index e1a1228..09191e0 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/bean/BeanTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/bean/BeanTest.java
@@ -54,7 +54,7 @@ public class BeanTest {
} catch (IntrospectionException e) {
}
- new SQLResponse(null, null, null, 0, true, null, false, false);
+ new SQLResponse(null, null, 0, true, null);
SelectedColumnMeta coulmnMeta = new SelectedColumnMeta(false, false, false, false, 0, false, 0, null, null,
null, null, null, 0, 0, 0, null, false, false, false);
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
index 7c5f253..2225096 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
@@ -31,7 +31,7 @@ import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
-import net.sf.ehcache.CacheManager;
+import org.springframework.cache.CacheManager;
/**
* @author xduo