You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/11/23 05:53:18 UTC
[16/18] kylin git commit: APACHE-KYLIN-2734: backend support for hot
cuboids export & import
APACHE-KYLIN-2734: backend support for hot cuboids export & import
Signed-off-by: Zhong <nj...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0aec7557
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0aec7557
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0aec7557
Branch: refs/heads/ci-dong
Commit: 0aec7557f71e6ee9f8f6bba5f780ef62f4677867
Parents: c6e8093
Author: Ma Gang <mg...@163.com>
Authored: Wed Aug 30 14:51:19 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Nov 23 13:31:34 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/model/CubeDesc.java | 41 +++++++++++++++++++-
.../kylin/rest/controller/CubeController.java | 8 ++--
.../apache/kylin/rest/service/BasicService.java | 1 -
.../apache/kylin/rest/service/QueryService.java | 7 ++--
4 files changed, 49 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0aec7557/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 01dd302..9c0a7cc 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -58,13 +58,13 @@ import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IEngineAware;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -185,6 +185,10 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
@JsonInclude(JsonInclude.Include.NON_NULL)
private int parentForward = 3;
+ @JsonProperty("mandatory_dimension_set_list")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private List<Set<String>> mandatoryDimensionSetList = Collections.emptyList();
+
private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<>();
private LinkedHashSet<ColumnDesc> allColumnDescs = new LinkedHashSet<>();
private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<>();
@@ -446,6 +450,14 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
this.overrideKylinProps = overrideKylinProps;
}
+ public List<Set<String>> getMandatoryDimensionSetList() {
+ return mandatoryDimensionSetList;
+ }
+
+ public void setMandatoryDimensionSetList(List<Set<String>> mandatoryDimensionSetList) {
+ this.mandatoryDimensionSetList = mandatoryDimensionSetList;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o)
@@ -537,6 +549,13 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
.append(JsonUtil.writeValueAsString(this.engineType)).append("|")//
.append(JsonUtil.writeValueAsString(this.storageType)).append("|");
+ if (mandatoryDimensionSetList != null && !mandatoryDimensionSetList.isEmpty()) {
+ for (Set<String> mandatoryDimensionSet : mandatoryDimensionSetList) {
+ TreeSet<String> sortedSet = Sets.newTreeSet(mandatoryDimensionSet);
+ sigString.append(JsonUtil.writeValueAsString(sortedSet)).append("|");
+ }
+ }
+
String signatureInput = sigString.toString().replaceAll("\\s+", "").toLowerCase();
byte[] signature = md.digest(signatureInput.getBytes());
@@ -631,6 +650,26 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
initDictionaryDesc();
amendAllColumns();
+
+ // check if mandatory dimension set list is valid
+ validateMandatoryDimensionSetList();
+ }
+
+ public void validateMandatoryDimensionSetList() {
+ Set<String> rowKeyColumns = Sets.newHashSet();
+ for (RowKeyColDesc entry : getRowkey().getRowKeyColumns()) {
+ rowKeyColumns.add(entry.getColumn());
+ }
+
+ for (Set<String> mandatoryDimensionSet : this.mandatoryDimensionSetList) {
+ for (String columnName : mandatoryDimensionSet) {
+ if (!rowKeyColumns.contains(columnName)) {
+ logger.info("Column " + columnName + " in " + mandatoryDimensionSet + " does not exist");
+ throw new IllegalStateException(
+ "Column " + columnName + " in " + mandatoryDimensionSet + " does not exist");
+ }
+ }
+ }
}
public CuboidScheduler getInitialCuboidScheduler() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0aec7557/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index aa59e30..0e9f4ba 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -898,11 +898,13 @@ public class CubeController extends BasicController {
return getCuboidHitFrequency(cubeName, false);
}
- private Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean ifSource) {
+ private Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean isCuboidSource) {
SQLRequest sqlRequest = new SQLRequest();
sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
- String cuboidColumn = ifSource ? QueryCubePropertyEnum.CUBOID_SOURCE.toString()
- : QueryCubePropertyEnum.CUBOID_TARGET.toString();
+ String cuboidColumn = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
+ if (!isCuboidSource) {
+ cuboidColumn = QueryCubePropertyEnum.CUBOID_TARGET.toString();
+ }
String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
String table = cubeService.getMetricsManager()
.getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
http://git-wip-us.apache.org/repos/asf/kylin/blob/0aec7557/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
index f8f50f3..9ac2602 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -98,5 +98,4 @@ public abstract class BasicService {
public MetricsManager getMetricsManager() {
return MetricsManager.getInstance();
}
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0aec7557/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 3f16646..9c3d34f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -464,7 +464,7 @@ public class QueryService extends BasicService {
sqlResponse.setTotalScanBytes(0);
}
- checkQueryAuth(sqlResponse, project);
+ checkQueryAuth(sqlResponse, project, secureEnabled);
} catch (Throwable e) { // calcite may throw AssertError
logger.error("Exception while executing query", e);
@@ -526,8 +526,9 @@ public class QueryService extends BasicService {
return response;
}
- protected void checkQueryAuth(SQLResponse sqlResponse, String project) throws AccessDeniedException {
- if (!sqlResponse.getIsException() && KylinConfig.getInstanceFromEnv().isQuerySecureEnabled()) {
+ protected void checkQueryAuth(SQLResponse sqlResponse, String project, boolean secureEnabled)
+ throws AccessDeniedException {
+ if (!sqlResponse.getIsException() && KylinConfig.getInstanceFromEnv().isQuerySecureEnabled() && secureEnabled) {
checkAuthorization(sqlResponse, project);
}
}