You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by nj...@apache.org on 2017/12/02 17:24:07 UTC
[14/19] kylin git commit: APACHE-KYLIN-2822: backend support for
sunburst chart to show cuboid tree
APACHE-KYLIN-2822: backend support for sunburst chart to show cuboid tree
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/75f6b005
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/75f6b005
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/75f6b005
Branch: refs/heads/master
Commit: 75f6b0054e78c6abb5a1723161b04fbd9acac4a8
Parents: 6236bfd
Author: Zhong <nj...@apache.org>
Authored: Wed Aug 30 18:04:09 2017 +0800
Committer: Zhong <nj...@apache.org>
Committed: Sat Dec 2 23:43:43 2017 +0800
----------------------------------------------------------------------
.../kylin/cube/cuboid/TreeCuboidScheduler.java | 3 +-
.../engine/mr/common/CuboidStatsReaderUtil.java | 10 +-
.../kylin/rest/controller/CubeController.java | 181 +++++++++++++++++++
.../kylin/rest/response/CuboidTreeResponse.java | 123 +++++++++++++
.../apache/kylin/rest/service/CubeService.java | 87 +++++++++
5 files changed, 399 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/75f6b005/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
index 903e358..45b741e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
@@ -19,7 +19,6 @@
package org.apache.kylin.cube.cuboid;
import java.io.PrintWriter;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -244,7 +243,7 @@ public class TreeCuboidScheduler extends CuboidScheduler {
@JsonIgnore
int level;
@JsonProperty("children")
- List<TreeNode> children = new ArrayList<>();
+ List<TreeNode> children = Lists.newArrayList();
public long getCuboidId() {
return cuboidId;
http://git-wip-us.apache.org/repos/asf/kylin/blob/75f6b005/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
index 56ab504..1542aa2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
@@ -38,9 +38,13 @@ public class CuboidStatsReaderUtil {
private static final Logger logger = LoggerFactory.getLogger(CuboidStatsReaderUtil.class);
- public static Map<Long, Long> readCuboidStatsFromCube(Set<Long> cuboidIds, CubeInstance cubeInstance)
- throws IOException {
- Map<Long, Long> statisticsMerged = readCuboidStatsAndSizeFromCube(cuboidIds, cubeInstance).getFirst();
+ public static Map<Long, Long> readCuboidStatsFromCube(Set<Long> cuboidIds, CubeInstance cubeInstance) {
+ Map<Long, Long> statisticsMerged = null;
+ try {
+ statisticsMerged = readCuboidStatsAndSizeFromCube(cuboidIds, cubeInstance).getFirst();
+ } catch (IOException e) {
+ logger.warn("Fail to read statistics for cube " + cubeInstance.getName() + " due to " + e);
+ }
return statisticsMerged.isEmpty() ? null : statisticsMerged;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/75f6b005/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 77bd498..aa59e30 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -21,21 +21,30 @@ package org.apache.kylin.rest.controller;
import static org.apache.kylin.rest.service.CubeService.VALID_CUBENAME;
import java.io.IOException;
+import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
+import javax.servlet.http.HttpServletResponse;
+
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.cuboid.TreeCuboidScheduler;
import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.RowKeyColDesc;
import org.apache.kylin.dimension.DimensionEncodingFactory;
import org.apache.kylin.engine.EngineFactory;
+import org.apache.kylin.engine.mr.common.CuboidStatsReaderUtil;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.JoinedFlatTable;
import org.apache.kylin.job.exception.JobException;
@@ -45,6 +54,8 @@ import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentRange.TSRange;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.metrics.MetricsManager;
+import org.apache.kylin.metrics.property.QueryCubePropertyEnum;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -53,6 +64,8 @@ import org.apache.kylin.rest.request.CubeRequest;
import org.apache.kylin.rest.request.JobBuildRequest;
import org.apache.kylin.rest.request.JobBuildRequest2;
import org.apache.kylin.rest.request.JobOptimizeRequest;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.response.GeneralResponse;
import org.apache.kylin.rest.response.HBaseResponse;
@@ -60,6 +73,7 @@ import org.apache.kylin.rest.response.ResponseCode;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
import org.apache.kylin.rest.service.ProjectService;
+import org.apache.kylin.rest.service.QueryService;
import org.apache.kylin.source.kafka.util.KafkaClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -81,6 +95,7 @@ import com.fasterxml.jackson.databind.JsonMappingException;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
/**
* CubeController is defined as Restful API entrance for UI.
@@ -102,6 +117,10 @@ public class CubeController extends BasicController {
@Qualifier("projectService")
private ProjectService projectService;
+ @Autowired
+ @Qualifier("queryService")
+ private QueryService queryService;
+
@RequestMapping(value = "/validate/{cubeName}", method = RequestMethod.GET, produces = { "application/json" })
@ResponseBody
public EnvelopeResponse<Boolean> validateModelName(@PathVariable String cubeName) {
@@ -768,6 +787,168 @@ public class CubeController extends BasicController {
}
+ @RequestMapping(value = "/{cubeName}/cuboids/export", method = RequestMethod.GET)
+ @ResponseBody
+ public void cuboidsExport(@PathVariable String cubeName, @RequestParam(value = "top") Integer top,
+ HttpServletResponse response) throws IOException {
+ CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+ if (cube == null) {
+ logger.error("Get cube: [" + cubeName + "] failed when get recommend cuboids");
+ throw new BadRequestException("Get cube: [" + cubeName + "] failed when get recommend cuboids");
+ }
+ Map<Long, Long> cuboidList = getRecommendCuboidList(cube);
+ if (cuboidList == null || cuboidList.isEmpty()) {
+ logger.warn("Cannot get recommend cuboid list for cube " + cubeName);
+ }
+ if (cuboidList.size() < top) {
+ logger.info("Only recommend " + cuboidList.size() + " cuboids less than topn " + top);
+ }
+ Iterator<Long> cuboidIterator = cuboidList.keySet().iterator();
+ RowKeyColDesc[] rowKeyColDescList = cube.getDescriptor().getRowkey().getRowKeyColumns();
+
+ List<Set<String>> dimensionSetList = Lists.newLinkedList();
+ while (top-- > 0 && cuboidIterator.hasNext()) {
+ Set<String> dimensionSet = Sets.newHashSet();
+ dimensionSetList.add(dimensionSet);
+ long cuboid = cuboidIterator.next();
+ for (int i = 0; i < rowKeyColDescList.length; i++) {
+ if ((cuboid & (1L << rowKeyColDescList[i].getBitIndex())) > 0) {
+ dimensionSet.add(rowKeyColDescList[i].getColumn());
+ }
+ }
+ }
+
+ response.setContentType("text/json;charset=utf-8");
+ response.setHeader("Content-Disposition", "attachment; filename=\"" + cubeName + ".json\"");
+ try (PrintWriter writer = response.getWriter()) {
+ writer.write(JsonUtil.writeValueAsString(dimensionSetList));
+ } catch (IOException e) {
+ logger.error("", e);
+ throw new InternalErrorException("Failed to write: " + e.getLocalizedMessage());
+ }
+ }
+
+ @RequestMapping(value = "/{cubeName}/cuboids/current", method = RequestMethod.GET)
+ @ResponseBody
+ public CuboidTreeResponse getCurrentCuboids(@PathVariable String cubeName) {
+ CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+ if (cube == null) {
+ logger.error("Get cube: [" + cubeName + "] failed when get current cuboids");
+ throw new BadRequestException("Get cube: [" + cubeName + "] failed when get current cuboids");
+ }
+ // The cuboid tree displayed should be consistent with the current one
+ CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+ Map<Long, Long> cuboidStatsMap = cube.getCuboids();
+ if (cuboidStatsMap == null) {
+ cuboidStatsMap = CuboidStatsReaderUtil.readCuboidStatsFromCube(cuboidScheduler.getAllCuboidIds(), cube);
+ }
+
+ Map<Long, Long> hitFrequencyMap = null;
+ Map<Long, Long> queryMatchMap = null;
+ try {
+ hitFrequencyMap = getTargetCuboidHitFrequency(cubeName);
+ queryMatchMap = getCuboidQueryMatchCount(cubeName);
+ } catch (Exception e) {
+ logger.warn("Fail to query on system cube due to " + e);
+ }
+
+ Set<Long> currentCuboidSet = cube.getCuboidScheduler().getAllCuboidIds();
+ return cubeService.getCuboidTreeResponse(cuboidScheduler, cuboidStatsMap, hitFrequencyMap, queryMatchMap,
+ currentCuboidSet);
+ }
+
+ @RequestMapping(value = "/{cubeName}/cuboids/recommend", method = RequestMethod.GET)
+ @ResponseBody
+ public CuboidTreeResponse getRecommendCuboids(@PathVariable String cubeName) throws IOException {
+ CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+ if (cube == null) {
+ logger.error("Get cube: [" + cubeName + "] failed when get recommend cuboids");
+ throw new BadRequestException("Get cube: [" + cubeName + "] failed when get recommend cuboids");
+ }
+ Map<Long, Long> recommendCuboidStatsMap = getRecommendCuboidList(cube);
+ if (recommendCuboidStatsMap == null || recommendCuboidStatsMap.isEmpty()) {
+ return new CuboidTreeResponse();
+ }
+ CuboidScheduler cuboidScheduler = new TreeCuboidScheduler(cube.getDescriptor(),
+ Lists.newArrayList(recommendCuboidStatsMap.keySet()),
+ new TreeCuboidScheduler.CuboidCostComparator(recommendCuboidStatsMap));
+
+ // Get cuboid target info for displaying heat map of cuboid hit
+ Map<Long, Long> displayHitFrequencyMap = getTargetCuboidHitFrequency(cubeName);
+ // Get exactly matched cuboid query count
+ Map<Long, Long> queryMatchMap = getCuboidQueryMatchCount(cubeName);
+
+ Set<Long> currentCuboidSet = cube.getCuboidScheduler().getAllCuboidIds();
+ return cubeService.getCuboidTreeResponse(cuboidScheduler, recommendCuboidStatsMap, displayHitFrequencyMap,
+ queryMatchMap, currentCuboidSet);
+ }
+
+ private Map<Long, Long> getRecommendCuboidList(CubeInstance cube) throws IOException {
+ // Get cuboid source info
+ Map<Long, Long> optimizeHitFrequencyMap = getSourceCuboidHitFrequency(cube.getName());
+ Map<Long, Map<Long, Long>> rollingUpCountSourceMap = getCuboidRollingUpCount(cube.getName());
+ return cubeService.getRecommendCuboidStatistics(cube, optimizeHitFrequencyMap, rollingUpCountSourceMap);
+ }
+
+ private Map<Long, Long> getSourceCuboidHitFrequency(String cubeName) {
+ return getCuboidHitFrequency(cubeName, true);
+ }
+
+ private Map<Long, Long> getTargetCuboidHitFrequency(String cubeName) {
+ return getCuboidHitFrequency(cubeName, false);
+ }
+
+ private Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean ifSource) {
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ String cuboidColumn = ifSource ? QueryCubePropertyEnum.CUBOID_SOURCE.toString()
+ : QueryCubePropertyEnum.CUBOID_TARGET.toString();
+ String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
+ String table = cubeService.getMetricsManager()
+ .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
+ String sql = "select " + cuboidColumn + ", sum(" + hitMeasure + ") " //
+ + "from " + table//
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " //
+ + "group by " + cuboidColumn;
+ sqlRequest.setSql(sql);
+ List<List<String>> orgHitFrequency = queryService.queryWithoutSecure(sqlRequest).getResults();
+ return cubeService.formatQueryCount(orgHitFrequency);
+ }
+
+ private Map<Long, Map<Long, Long>> getCuboidRollingUpCount(String cubeName) {
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
+ String cuboidTarget = QueryCubePropertyEnum.CUBOID_TARGET.toString();
+ String aggCount = QueryCubePropertyEnum.AGGR_COUNT.toString();
+ String table = cubeService.getMetricsManager()
+ .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
+ String sql = "select " + cuboidSource + ", " + cuboidTarget + ", sum(" + aggCount + ")/count(*) " //
+ + "from " + table //
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " //
+ + "group by " + cuboidSource + ", " + cuboidTarget;
+ sqlRequest.setSql(sql);
+ List<List<String>> orgRollingUpCount = queryService.queryWithoutSecure(sqlRequest).getResults();
+ return cubeService.formatRollingUpCount(orgRollingUpCount);
+ }
+
+ private Map<Long, Long> getCuboidQueryMatchCount(String cubeName) {
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
+ String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
+ String table = cubeService.getMetricsManager()
+ .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
+ String sql = "select " + cuboidSource + ", sum(" + hitMeasure + ") " //
+ + "from " + table //
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' and "
+ + QueryCubePropertyEnum.IF_MATCH.toString() + " = true " //
+ + "group by " + cuboidSource;
+ sqlRequest.setSql(sql);
+ List<List<String>> orgMatchHitFrequency = queryService.queryWithoutSecure(sqlRequest).getResults();
+ return cubeService.formatQueryCount(orgMatchHitFrequency);
+ }
+
/**
* Initiate the very beginning of a streaming cube. Will seek the latest offests of each partition from streaming
* source (kafka) and record in the cube descriptor; In the first build job, it will use these offests as the start point.
http://git-wip-us.apache.org/repos/asf/kylin/blob/75f6b005/server-base/src/main/java/org/apache/kylin/rest/response/CuboidTreeResponse.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/CuboidTreeResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/CuboidTreeResponse.java
new file mode 100644
index 0000000..b416084
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/CuboidTreeResponse.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.response;
+
+import java.io.Serializable;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Lists;
+
+public class CuboidTreeResponse implements Serializable {
+
+ private static final long serialVersionUID = 2835980715891990832L;
+
+ private NodeInfo root;
+
+ public NodeInfo getRoot() {
+ return root;
+ }
+
+ public void setRoot(NodeInfo root) {
+ this.root = root;
+ }
+
+ public static class NodeInfo {
+ @JsonProperty("cuboid_id")
+ private Long id;
+ @JsonProperty("name")
+ private String name;
+ @JsonProperty("query_count")
+ private Long queryCount;
+ @JsonProperty("query_rate")
+ private Float queryRate;
+ @JsonProperty("exactly_match_count")
+ private Long exactlyMatchCount;
+ @JsonProperty("row_count")
+ private Long rowCount;
+ @JsonProperty("existed")
+ private Boolean existed;
+ @JsonProperty("children")
+ List<NodeInfo> children = Lists.newArrayList();
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public Long getQueryCount() {
+ return queryCount;
+ }
+
+ public void setQueryCount(Long queryCount) {
+ this.queryCount = queryCount;
+ }
+
+ public Float getQueryRate() {
+ return queryRate;
+ }
+
+ public void setQueryRate(Float queryRate) {
+ this.queryRate = queryRate;
+ }
+
+ public Long getExactlyMatchCount() {
+ return exactlyMatchCount;
+ }
+
+ public void setExactlyMatchCount(Long exactlyMatchCount) {
+ this.exactlyMatchCount = exactlyMatchCount;
+ }
+
+ public Long getRowCount() {
+ return rowCount;
+ }
+
+ public void setRowCount(Long rowCount) {
+ this.rowCount = rowCount;
+ }
+
+ public Boolean getExisted() {
+ return existed;
+ }
+
+ public void setExisted(Boolean existed) {
+ this.existed = existed;
+ }
+
+ public void addChild(NodeInfo child) {
+ this.children.add(child);
+ }
+
+ public List<NodeInfo> getChildren() {
+ return children;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/75f6b005/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index d5805a1..b86ff1d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -24,6 +24,8 @@ import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -32,10 +34,13 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.CubeUpdate;
+import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidCLI;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.engine.mr.common.CuboidRecommenderUtil;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.job.execution.ExecutableState;
@@ -55,6 +60,8 @@ import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
import org.apache.kylin.rest.request.MetricsRequest;
+import org.apache.kylin.rest.response.CuboidTreeResponse;
+import org.apache.kylin.rest.response.CuboidTreeResponse.NodeInfo;
import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.security.AclPermission;
@@ -761,4 +768,84 @@ public class CubeService extends BasicService implements InitializingBean {
}
}
}
+
+ public CuboidTreeResponse getCuboidTreeResponse(CuboidScheduler cuboidScheduler, Map<Long, Long> rowCountMap,
+ Map<Long, Long> hitFrequencyMap, Map<Long, Long> queryMatchMap, Set<Long> currentCuboidSet) {
+ long baseCuboidId = cuboidScheduler.getBaseCuboidId();
+ int dimensionCount = Long.bitCount(baseCuboidId);
+
+ // get cube query count total
+ long cubeQueryCount = 0L;
+ if (hitFrequencyMap != null) {
+ for (long queryCount : hitFrequencyMap.values()) {
+ cubeQueryCount += queryCount;
+ }
+ }
+
+ NodeInfo root = generateNodeInfo(baseCuboidId, dimensionCount, cubeQueryCount, rowCountMap, hitFrequencyMap,
+ queryMatchMap, currentCuboidSet);
+
+ List<NodeInfo> nodeQueue = Lists.newLinkedList();
+ nodeQueue.add(root);
+ while (!nodeQueue.isEmpty()) {
+ NodeInfo parentNode = nodeQueue.remove(0);
+ for (long childId : cuboidScheduler.getSpanningCuboid(parentNode.getId())) {
+ NodeInfo childNode = generateNodeInfo(childId, dimensionCount, cubeQueryCount, rowCountMap,
+ hitFrequencyMap, queryMatchMap, currentCuboidSet);
+ parentNode.addChild(childNode);
+ nodeQueue.add(childNode);
+ }
+ }
+
+ CuboidTreeResponse result = new CuboidTreeResponse();
+ result.setRoot(root);
+ return result;
+ }
+
+ private NodeInfo generateNodeInfo(long cuboidId, int dimensionCount, long cubeQueryCount,
+ Map<Long, Long> rowCountMap, Map<Long, Long> hitFrequencyMap, Map<Long, Long> queryMatchMap,
+ Set<Long> currentCuboidSet) {
+ Long queryCount = hitFrequencyMap == null || hitFrequencyMap.get(cuboidId) == null ? 0L
+ : hitFrequencyMap.get(cuboidId);
+ float queryRate = cubeQueryCount <= 0 ? 0 : queryCount.floatValue() / cubeQueryCount;
+ long queryExactlyMatchCount = queryMatchMap == null || queryMatchMap.get(cuboidId) == null ? 0L
+ : queryMatchMap.get(cuboidId);
+ boolean ifExist = currentCuboidSet.contains(cuboidId);
+ long rowCount = rowCountMap == null ? 0L : rowCountMap.get(cuboidId);
+
+ NodeInfo node = new NodeInfo();
+ node.setId(cuboidId);
+ node.setName(Cuboid.getDisplayName(cuboidId, dimensionCount));
+ node.setQueryCount(queryCount);
+ node.setQueryRate(queryRate);
+ node.setExactlyMatchCount(queryExactlyMatchCount);
+ node.setExisted(ifExist);
+ node.setRowCount(rowCount);
+ return node;
+ }
+
+ /** cube planner services */
+ public Map<Long, Long> formatQueryCount(List<List<String>> orgQueryCount) {
+ Map<Long, Long> formattedQueryCount = Maps.newLinkedHashMap();
+ for (List<String> hit : orgQueryCount) {
+ formattedQueryCount.put(Long.parseLong(hit.get(0)), (long) Double.parseDouble(hit.get(1)));
+ }
+ return formattedQueryCount;
+ }
+
+ public Map<Long, Map<Long, Long>> formatRollingUpCount(List<List<String>> orgRollingUpCount) {
+ Map<Long, Map<Long, Long>> formattedRollingUpCount = Maps.newLinkedHashMap();
+ for (List<String> rollingUp : orgRollingUpCount) {
+ Map<Long, Long> childMap = Maps.newLinkedHashMap();
+ childMap.put(Long.parseLong(rollingUp.get(1)), (long) Double.parseDouble(rollingUp.get(2)));
+ formattedRollingUpCount.put(Long.parseLong(rollingUp.get(0)), childMap);
+ }
+ return formattedRollingUpCount;
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION')")
+ public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
+ Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
+ return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap);
+ }
}