You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by nj...@apache.org on 2018/12/25 01:29:03 UTC
[kylin] branch master updated (0c92d1c -> 4850dac)
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 0c92d1c minor, fix format in pom
new 8cfe32c KYLIN-3540 move queryService of CubeController to CubeService
new 4db6a37 KYLIN-3540 refactor the interface of querying on SYSTEM project
new 4850dac KYLIN-3540 estimate the row counts of source cuboids which are not built & remove mandatory cuboids recommendation
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/kylin/common/KylinConfigBase.java | 18 +-
.../cube/cuboid/algorithm/BPUSCalculator.java | 2 +-
.../cube/cuboid/algorithm/CuboidRecommender.java | 25 ++-
.../kylin/cube/cuboid/algorithm/CuboidStats.java | 131 ++++++------
.../cube/cuboid/algorithm/CuboidStatsUtil.java | 226 ++++++++++++++++-----
.../cube/cuboid/algorithm/CuboidStatsUtilTest.java | 162 ++++++++++++---
.../engine/mr/common/CuboidRecommenderUtil.java | 44 +++-
.../engine/mr/common/CuboidStatsReaderUtil.java | 23 +++
.../kylin/rest/controller/CubeController.java | 73 +------
.../kylin/rest/controller/DashboardController.java | 66 +++---
.../org/apache/kylin/rest/service/CubeService.java | 85 ++++++--
.../kylin/rest/service/DashboardService.java | 34 ++--
.../apache/kylin/rest/service/QueryService.java | 14 +-
13 files changed, 617 insertions(+), 286 deletions(-)
[kylin] 03/03: KYLIN-3540 estimate the row counts of source cuboids
which are not built & remove mandatory cuboids recommendation
Posted by nj...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4850dacece8a2e90f4333b50e2e8304635a730a2
Author: kyotoYaho <nj...@apache.org>
AuthorDate: Tue Dec 18 18:54:51 2018 +0800
KYLIN-3540 estimate the row counts of source cuboids which are not built & remove mandatory cuboids recommendation
---
.../org/apache/kylin/common/KylinConfigBase.java | 18 +-
.../cube/cuboid/algorithm/BPUSCalculator.java | 2 +-
.../cube/cuboid/algorithm/CuboidRecommender.java | 25 ++-
.../kylin/cube/cuboid/algorithm/CuboidStats.java | 131 ++++++------
.../cube/cuboid/algorithm/CuboidStatsUtil.java | 226 ++++++++++++++++-----
.../cube/cuboid/algorithm/CuboidStatsUtilTest.java | 162 ++++++++++++---
.../engine/mr/common/CuboidRecommenderUtil.java | 44 +++-
.../engine/mr/common/CuboidStatsReaderUtil.java | 23 +++
.../kylin/rest/controller/CubeController.java | 4 +-
.../org/apache/kylin/rest/service/CubeService.java | 26 ++-
10 files changed, 495 insertions(+), 166 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f67f6b3..b63062e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -316,8 +316,7 @@ abstract public class KylinConfigBase implements Serializable {
public String getReadHdfsWorkingDirectory() {
if (StringUtils.isNotEmpty(getHBaseClusterFs())) {
Path workingDir = new Path(getHdfsWorkingDirectory());
- return new Path(getHBaseClusterFs(), Path.getPathWithoutSchemeAndAuthority(workingDir)).toString()
- + "/";
+ return new Path(getHBaseClusterFs(), Path.getPathWithoutSchemeAndAuthority(workingDir)).toString() + "/";
}
return getHdfsWorkingDirectory();
@@ -644,8 +643,12 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.cube.cubeplanner.recommend-cache-max-size", "200"));
}
- public long getCubePlannerMandatoryRollUpThreshold() {
- return Long.parseLong(getOptional("kylin.cube.cubeplanner.mandatory-rollup-threshold", "1000"));
+ public double getCubePlannerQueryUncertaintyRatio() {
+ return Double.parseDouble(getOptional("kylin.cube.cubeplanner.query-uncertainty-ratio", "0.1"));
+ }
+
+ public double getCubePlannerBPUSMinBenefitRatio() {
+ return Double.parseDouble(getOptional("kylin.cube.cubeplanner.bpus-min-benefit-ratio", "0.01"));
}
public int getCubePlannerAgreedyAlgorithmAutoThreshold() {
@@ -1910,12 +1913,13 @@ abstract public class KylinConfigBase implements Serializable {
}
public int getSmallCellMetadataWarningThreshold() {
- return Integer.parseInt(getOptional("kylin.metadata.jdbc.small-cell-meta-size-warning-threshold",
- String.valueOf(100 << 20))); //100mb
+ return Integer.parseInt(
+ getOptional("kylin.metadata.jdbc.small-cell-meta-size-warning-threshold", String.valueOf(100 << 20))); //100mb
}
public int getSmallCellMetadataErrorThreshold() {
- return Integer.parseInt(getOptional("kylin.metadata.jdbc.small-cell-meta-size-error-threshold", String.valueOf(1 << 30))); // 1gb
+ return Integer.parseInt(
+ getOptional("kylin.metadata.jdbc.small-cell-meta-size-error-threshold", String.valueOf(1 << 30))); // 1gb
}
public int getJdbcResourceStoreMaxCellSize() {
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
index 6316858..39c52da 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
@@ -142,7 +142,7 @@ public class BPUSCalculator implements BenefitPolicy {
}
public double getMinBenefitRatio() {
- return 0.01;
+ return cuboidStats.getBpusMinBenefitRatio();
}
@Override
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
index baacb51..0e6a844 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
@@ -154,12 +154,11 @@ public class CuboidRecommender {
Map<Long, Long> recommendCuboidsWithStats = Maps.newLinkedHashMap();
for (Long cuboid : recommendCuboidList) {
- if (cuboid.equals(cuboidStats.getBaseCuboid())) {
- recommendCuboidsWithStats.put(cuboid, cuboidStats.getCuboidCount(cuboid));
- } else if (cuboidStats.getAllCuboidsForSelection().contains(cuboid)) {
- recommendCuboidsWithStats.put(cuboid, cuboidStats.getCuboidCount(cuboid));
+ if (cuboid == 0L) {
+ // for zero cuboid, just simply recommend the cheapest cuboid.
+ handleCuboidZeroRecommend(cuboidStats, recommendCuboidsWithStats);
} else {
- recommendCuboidsWithStats.put(cuboid, -1L);
+ recommendCuboidsWithStats.put(cuboid, cuboidStats.getCuboidCount(cuboid));
}
}
@@ -168,4 +167,20 @@ public class CuboidRecommender {
}
return recommendCuboidsWithStats;
}
+
+ private void handleCuboidZeroRecommend(CuboidStats cuboidStats, Map<Long, Long> recommendCuboidsWithStats) {
+ Map<Long, Long> statistics = cuboidStats.getStatistics();
+ Long cheapestCuboid = null;
+ Long cheapestCuboidCount = Long.MAX_VALUE;
+ for (Map.Entry<Long, Long> cuboidStatsEntry : statistics.entrySet()) {
+ if (cuboidStatsEntry.getValue() < cheapestCuboidCount) {
+ cheapestCuboid = cuboidStatsEntry.getKey();
+ cheapestCuboidCount = cuboidStatsEntry.getValue();
+ }
+ }
+ if (cheapestCuboid != null) {
+ logger.info("recommend cuboid:{} instead of cuboid zero", cheapestCuboid);
+ recommendCuboidsWithStats.put(cheapestCuboid, cheapestCuboidCount);
+ }
+ }
}
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
index 78a6c5b..c22ab9a 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
@@ -18,57 +18,72 @@
package org.apache.kylin.cube.cuboid.algorithm;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.common.util.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
public class CuboidStats {
private static final Logger logger = LoggerFactory.getLogger(CuboidStats.class);
+ static final double WEIGHT_FOR_UN_QUERY = 0.2;
+ static final double BPUS_MIN_BENEFIT_RATIO = 0.001;
+
public static class Builder {
private static final long THRESHOLD_ROLL_UP_FOR_MANDATORY = 1000L;
// Required parameters
private String key;
+ private Long nTotalCuboids;
private Long baseCuboid;
+ private double queryUncertaintyRatio = WEIGHT_FOR_UN_QUERY;
+ private double bpusMinBenefitRatio = BPUS_MIN_BENEFIT_RATIO;
private Map<Long, Long> statistics;
private Map<Long, Double> size;
// Optional parameters - initialized to default values
private Set<Long> mandatoryCuboids = null;
//// These two properties are for generating mandatory cuboids
- private Map<Long, Map<Long, Long>> rollingUpCountSourceMap = null;
- private Long rollUpThresholdForMandatory = null;
+ private Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap = null;
private Map<Long, Long> hitFrequencyMap = null;
private Map<Long, Map<Long, Long>> scanCountSourceMap = null;
public Builder(String key, Long baseCuboid, Map<Long, Long> statistics, Map<Long, Double> size) {
+ this(key, baseCuboid, baseCuboid, statistics, size);
+ }
+
+ public Builder(String key, Long nTotalCuboids, Long baseCuboid, Map<Long, Long> statistics,
+ Map<Long, Double> size) {
this.key = key;
+ this.nTotalCuboids = nTotalCuboids;
this.baseCuboid = baseCuboid;
this.statistics = statistics;
this.size = size;
}
- public Builder setRollingUpCountSourceMap(Map<Long, Map<Long, Long>> rollingUpCountSourceMap) {
- this.rollingUpCountSourceMap = rollingUpCountSourceMap;
- this.rollUpThresholdForMandatory = THRESHOLD_ROLL_UP_FOR_MANDATORY;
+ public Builder setQueryUncertaintyRatio(double queryUncertaintyRatio) {
+ this.queryUncertaintyRatio = queryUncertaintyRatio;
return this;
}
- public Builder setRollingUpCountSourceMap(Map<Long, Map<Long, Long>> rollingUpCountSourceMap,
- long rollUpThresholdForMandatory) {
+ public Builder setBPUSMinBenefitRatio(double bpusMinBenefitRatio) {
+ this.bpusMinBenefitRatio = bpusMinBenefitRatio;
+ return this;
+ }
+
+ public Builder setRollingUpCountSourceMap(Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap) {
this.rollingUpCountSourceMap = rollingUpCountSourceMap;
- this.rollUpThresholdForMandatory = rollUpThresholdForMandatory;
return this;
}
@@ -87,6 +102,10 @@ public class CuboidStats {
return this;
}
+ public Map<Long, Double> estimateCuboidsSize(Map<Long, Long> statistics) {
+ return null;
+ }
+
public CuboidStats build() {
Preconditions.checkNotNull(key, "key should not be null");
Preconditions.checkNotNull(baseCuboid, "baseCuboid should not be null");
@@ -96,23 +115,36 @@ public class CuboidStats {
"row count should exist for base cuboid " + baseCuboid);
Preconditions.checkState(statistics.keySet().equals(size.keySet()),
"statistics & size should own the same key set");
+ statistics = CuboidStatsUtil.adjustCuboidStats(statistics);
+
+ if (hitFrequencyMap != null && rollingUpCountSourceMap != null) {
+ Map<Long, Double> cuboidHitProbabilityMap = CuboidStatsUtil.calculateCuboidHitProbability(
+ hitFrequencyMap.keySet(), hitFrequencyMap, nTotalCuboids, queryUncertaintyRatio);
+ Map<Long, Long> srcCuboidsStats = CuboidStatsUtil.generateSourceCuboidStats(statistics,
+ cuboidHitProbabilityMap, rollingUpCountSourceMap);
+
+ statistics.putAll(srcCuboidsStats);
+
+ Map<Long, Double> estimatedSize = estimateCuboidsSize(statistics);
+ if (estimatedSize != null && !estimatedSize.isEmpty()) {
+ size = Maps.newHashMap(estimatedSize);
+ }
+ }
+
if (mandatoryCuboids == null) {
mandatoryCuboids = Sets.newHashSet();
- }
- if (rollingUpCountSourceMap != null) {
- mandatoryCuboids.addAll(CuboidStatsUtil.generateMandatoryCuboidSet(statistics, hitFrequencyMap,
- rollingUpCountSourceMap, rollUpThresholdForMandatory));
+ } else if (!mandatoryCuboids.isEmpty()) {
+ statistics.putAll(CuboidStatsUtil.complementRowCountForCuboids(statistics, mandatoryCuboids));
}
- return new CuboidStats(key, baseCuboid, mandatoryCuboids, statistics, size, hitFrequencyMap,
- scanCountSourceMap);
+ return new CuboidStats(key, baseCuboid, queryUncertaintyRatio, bpusMinBenefitRatio, mandatoryCuboids,
+ statistics, size, hitFrequencyMap, scanCountSourceMap);
}
}
- private static final double WEIGHT_FOR_UN_QUERY = 0.2;
-
private String key;
private long baseCuboid;
+ private double bpusMinBenefitRatio;
private ImmutableSet<Long> mandatoryCuboidSet;
private ImmutableSet<Long> selectionCuboidSet;
private ImmutableMap<Long, Long> cuboidCountMap;
@@ -123,11 +155,13 @@ public class CuboidStats {
private ImmutableMap<Long, List<Long>> directChildrenCache;
private Map<Long, Set<Long>> allDescendantsCache;
- private CuboidStats(String key, long baseCuboidId, Set<Long> mandatoryCuboids, Map<Long, Long> statistics,
- Map<Long, Double> size, Map<Long, Long> hitFrequencyMap, Map<Long, Map<Long, Long>> scanCountSourceMap) {
+ private CuboidStats(String key, long baseCuboidId, double queryUncertaintyRatio, double bpusMinBenefitRatio,
+ Set<Long> mandatoryCuboids, Map<Long, Long> statistics, Map<Long, Double> size,
+ Map<Long, Long> hitFrequencyMap, Map<Long, Map<Long, Long>> scanCountSourceMap) {
this.key = key;
this.baseCuboid = baseCuboidId;
+ this.bpusMinBenefitRatio = bpusMinBenefitRatio;
/** Initial mandatory cuboids */
Set<Long> cuboidsForMandatory = Sets.newHashSet(mandatoryCuboids);
//Always add base cuboid.
@@ -141,44 +175,19 @@ public class CuboidStats {
cuboidsForSelection.removeAll(cuboidsForMandatory);
//There's no overlap between mandatoryCuboidSet and selectionCuboidSet
- this.mandatoryCuboidSet = ImmutableSet.<Long>builder().addAll(cuboidsForMandatory).build();
- this.selectionCuboidSet = ImmutableSet.<Long>builder().addAll(cuboidsForSelection).build();
+ this.mandatoryCuboidSet = ImmutableSet.<Long> builder().addAll(cuboidsForMandatory).build();
+ this.selectionCuboidSet = ImmutableSet.<Long> builder().addAll(cuboidsForSelection).build();
if (selectionCuboidSet.isEmpty()) {
logger.warn("The selection set should not be empty!!!");
}
- /** Initialize row count for mandatory cuboids */
- CuboidStatsUtil.complementRowCountForMandatoryCuboids(statistics, baseCuboid, mandatoryCuboidSet);
-
- this.cuboidCountMap = ImmutableMap.<Long, Long>builder().putAll(statistics).build();
- this.cuboidSizeMap = ImmutableMap.<Long, Double>builder().putAll(size).build();
+ this.cuboidCountMap = ImmutableMap.<Long, Long> builder().putAll(statistics).build();
+ this.cuboidSizeMap = ImmutableMap.<Long, Double> builder().putAll(size).build();
/** Initialize the hit probability for each selection cuboid */
- Map<Long, Double> tmpCuboidHitProbabilityMap = Maps.newHashMapWithExpectedSize(selectionCuboidSet.size());
- if (hitFrequencyMap != null) {
- long totalHitFrequency = 0L;
- for (Map.Entry<Long, Long> hitFrequency : hitFrequencyMap.entrySet()) {
- if (selectionCuboidSet.contains(hitFrequency.getKey())) {
- totalHitFrequency += hitFrequency.getValue();
- }
- }
-
- final double unitUncertainProb = WEIGHT_FOR_UN_QUERY / selectionCuboidSet.size();
- for (Long cuboid : selectionCuboidSet) {
- //Calculate hit probability for each cuboid
- if (hitFrequencyMap.get(cuboid) != null) {
- tmpCuboidHitProbabilityMap.put(cuboid, unitUncertainProb
- + (1 - WEIGHT_FOR_UN_QUERY) * hitFrequencyMap.get(cuboid) / totalHitFrequency);
- } else {
- tmpCuboidHitProbabilityMap.put(cuboid, unitUncertainProb);
- }
- }
- } else {
- for (Long cuboid : selectionCuboidSet) {
- tmpCuboidHitProbabilityMap.put(cuboid, 1.0 / selectionCuboidSet.size());
- }
- }
- this.cuboidHitProbabilityMap = ImmutableMap.<Long, Double>builder().putAll(tmpCuboidHitProbabilityMap).build();
+ Map<Long, Double> tmpCuboidHitProbabilityMap = CuboidStatsUtil.calculateCuboidHitProbability(selectionCuboidSet,
+ hitFrequencyMap, selectionCuboidSet.size(), queryUncertaintyRatio);
+ this.cuboidHitProbabilityMap = ImmutableMap.<Long, Double> builder().putAll(tmpCuboidHitProbabilityMap).build();
/** Initialize the scan count when query for each selection cuboid + one base cuboid */
Map<Long, Long> tmpCuboidScanCountMap = Maps.newHashMapWithExpectedSize(1 + selectionCuboidSet.size());
@@ -186,16 +195,16 @@ public class CuboidStats {
for (Long cuboid : selectionCuboidSet) {
tmpCuboidScanCountMap.put(cuboid, getExpScanCount(cuboid, statistics, scanCountSourceMap));
}
- this.cuboidScanCountMap = ImmutableMap.<Long, Long>builder().putAll(tmpCuboidScanCountMap).build();
+ this.cuboidScanCountMap = ImmutableMap.<Long, Long> builder().putAll(tmpCuboidScanCountMap).build();
- this.directChildrenCache = ImmutableMap.<Long, List<Long>>builder()
+ this.directChildrenCache = ImmutableMap.<Long, List<Long>> builder()
.putAll(CuboidStatsUtil.createDirectChildrenCache(statistics.keySet())).build();
this.allDescendantsCache = Maps.newConcurrentMap();
}
private long getExpScanCount(long sourceCuboid, Map<Long, Long> statistics,
- Map<Long, Map<Long, Long>> scanCountSourceMap) {
+ Map<Long, Map<Long, Long>> scanCountSourceMap) {
Preconditions.checkNotNull(statistics.get(sourceCuboid),
"The statistics for source cuboid " + sourceCuboid + " does not exist!!!");
if (scanCountSourceMap == null || scanCountSourceMap.get(sourceCuboid) == null
@@ -216,6 +225,10 @@ public class CuboidStats {
}
}
+ public double getBpusMinBenefitRatio() {
+ return bpusMinBenefitRatio;
+ }
+
public Set<Long> getAllDescendants(long cuboid) {
Set<Long> allDescendants = Sets.newLinkedHashSet();
if (selectionCuboidSet.contains(cuboid)) {
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
index dc3471b..90eafdd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
@@ -27,81 +27,208 @@ import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+import org.apache.kylin.common.util.Pair;
+
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
public class CuboidStatsUtil {
/**
- * For generating mandatory cuboids,
- * a cuboid is mandatory if the expectation of rolling up count exceeds a threshold
+ * According to the cuboid hit frequencies and query uncertainty ratio
+ * calculate each cuboid hit probability
+ * @param selectionCuboidSet subset of cuboid domain which needs probability
+ * @param nTotalCuboids number of cuboids needs to be considered, mainly for each cuboid's uncertainty weight
* */
- public static Set<Long> generateMandatoryCuboidSet(Map<Long, Long> statistics, Map<Long, Long> hitFrequencyMap,
- Map<Long, Map<Long, Long>> rollingUpCountSourceMap, final long rollUpThresholdForMandatory) {
- Set<Long> mandatoryCuboidSet = Sets.newHashSet();
- if (hitFrequencyMap == null || hitFrequencyMap.isEmpty() || rollingUpCountSourceMap == null
- || rollingUpCountSourceMap.isEmpty()) {
- return mandatoryCuboidSet;
- }
- long totalHitFrequency = 0L;
- for (long hitFrequency : hitFrequencyMap.values()) {
- totalHitFrequency += hitFrequency;
+ public static Map<Long, Double> calculateCuboidHitProbability(Set<Long> selectionCuboidSet,
+ Map<Long, Long> hitFrequencyMap, long nTotalCuboids, double queryUncertaintyRatio) {
+ Map<Long, Double> cuboidHitProbabilityMap = Maps.newHashMapWithExpectedSize(selectionCuboidSet.size());
+ if (hitFrequencyMap == null || hitFrequencyMap.isEmpty()) {
+ for (Long cuboid : selectionCuboidSet) {
+ cuboidHitProbabilityMap.put(cuboid, 1.0 / nTotalCuboids);
+ }
+ } else {
+ long totalHitFrequency = 0L;
+ for (Map.Entry<Long, Long> hitFrequency : hitFrequencyMap.entrySet()) {
+ totalHitFrequency += hitFrequency.getValue();
+ }
+
+ final double unitUncertainProb = queryUncertaintyRatio / nTotalCuboids;
+ for (Long cuboid : selectionCuboidSet) {
+ //Calculate hit probability for each cuboid
+ if (hitFrequencyMap.get(cuboid) != null) {
+ cuboidHitProbabilityMap.put(cuboid, unitUncertainProb
+ + (1 - queryUncertaintyRatio) * hitFrequencyMap.get(cuboid) / totalHitFrequency);
+ } else {
+ cuboidHitProbabilityMap.put(cuboid, unitUncertainProb);
+ }
+ }
}
- if (totalHitFrequency == 0) {
- return mandatoryCuboidSet;
+ return cuboidHitProbabilityMap;
+ }
+
+ /**
+ * @param statistics for cuboid row count
+ * @param rollingUpSourceMap the key of the outer map is source cuboid,
+ * the key of the inner map is target cuboid,
+ * if cube is optimized multiple times, target cuboid may change
+ * the first element of the pair is the rollup row count
+ * the second element of the pair is the return row count
+ * @return source cuboids with estimated row count
+ */
+ public static Map<Long, Long> generateSourceCuboidStats(Map<Long, Long> statistics,
+ Map<Long, Double> cuboidHitProbabilityMap, Map<Long, Map<Long, Pair<Long, Long>>> rollingUpSourceMap) {
+ Map<Long, Long> srcCuboidsStats = Maps.newHashMap();
+ if (cuboidHitProbabilityMap == null || cuboidHitProbabilityMap.isEmpty() || rollingUpSourceMap == null
+ || rollingUpSourceMap.isEmpty()) {
+ return srcCuboidsStats;
}
- for (Map.Entry<Long, Long> hitFrequency : hitFrequencyMap.entrySet()) {
- long cuboid = hitFrequency.getKey();
+ for (Long cuboid : cuboidHitProbabilityMap.keySet()) {
+ if (statistics.get(cuboid) != null) {
+ continue;
+ }
+ Map<Long, Pair<Long, Long>> innerRollingUpTargetMap = rollingUpSourceMap.get(cuboid);
+ if (innerRollingUpTargetMap == null || innerRollingUpTargetMap.isEmpty()) {
+ continue;
+ }
- if (isCuboidMandatory(cuboid, statistics, rollingUpCountSourceMap)) {
- long totalEstScanCount = 0L;
- for (long estScanCount : rollingUpCountSourceMap.get(cuboid).values()) {
- totalEstScanCount += estScanCount;
- }
- totalEstScanCount /= rollingUpCountSourceMap.get(cuboid).size();
- if ((hitFrequency.getValue() * 1.0 / totalHitFrequency)
- * totalEstScanCount >= rollUpThresholdForMandatory) {
- mandatoryCuboidSet.add(cuboid);
+ long totalEstRowCount = 0L;
+ int nEffective = 0;
+ boolean ifHasStats = false;
+ // if ifHasStats equals true, then source cuboid row count = (1 - rollup ratio) * target cuboid row count
+ // else source cuboid row count = returned row count collected directly
+ for (Long tgtCuboid : innerRollingUpTargetMap.keySet()) {
+ Pair<Long, Long> rollingupStats = innerRollingUpTargetMap.get(tgtCuboid);
+ if (statistics.get(tgtCuboid) != null) {
+ if (!ifHasStats) {
+ totalEstRowCount = 0L;
+ nEffective = 0;
+ ifHasStats = true;
+ }
+ double rollupRatio = calculateRollupRatio(rollingupStats);
+ totalEstRowCount += (1 - rollupRatio) * statistics.get(tgtCuboid);
+ nEffective++;
+ } else {
+ if (ifHasStats) {
+ continue;
+ }
+ totalEstRowCount += rollingupStats.getSecond();
+ nEffective++;
}
}
- }
- return mandatoryCuboidSet;
- }
- private static boolean isCuboidMandatory(Long cuboid, Map<Long, Long> statistics, Map<Long, Map<Long, Long>> rollingUpCountSourceMap) {
- return !statistics.containsKey(cuboid) && rollingUpCountSourceMap.containsKey(cuboid) && !rollingUpCountSourceMap.get(cuboid).isEmpty();
+ srcCuboidsStats.put(cuboid, totalEstRowCount / nEffective);
+ }
+ srcCuboidsStats.remove(0L);
+ adjustCuboidStats(srcCuboidsStats, statistics);
+ return srcCuboidsStats;
}
/**
* Complement row count for mandatory cuboids
* with its best parent's row count
* */
- public static void complementRowCountForMandatoryCuboids(Map<Long, Long> statistics, long baseCuboid,
- Set<Long> mandatoryCuboidSet) {
+ public static Map<Long, Long> complementRowCountForCuboids(final Map<Long, Long> statistics, Set<Long> cuboids) {
+ Map<Long, Long> result = Maps.newHashMapWithExpectedSize(cuboids.size());
+
// Sort entries order by row count asc
- SortedSet<Map.Entry<Long, Long>> sortedStatsSet = new TreeSet<>(
- new Comparator<Map.Entry<Long, Long>>() {
- public int compare(Map.Entry<Long, Long> o1, Map.Entry<Long, Long> o2) {
- return o1.getValue().compareTo(o2.getValue());
- }
- });
- //sortedStatsSet.addAll(statistics.entrySet()); KYLIN-3580
- for(Map.Entry<Long, Long> entry : statistics.entrySet()){
- sortedStatsSet.add(entry);
- }
- for (Long cuboid : mandatoryCuboidSet) {
+ SortedSet<Map.Entry<Long, Long>> sortedStatsSet = new TreeSet<>(new Comparator<Map.Entry<Long, Long>>() {
+ public int compare(Map.Entry<Long, Long> o1, Map.Entry<Long, Long> o2) {
+ int ret = o1.getValue().compareTo(o2.getValue());
+ return ret == 0 ? o1.getKey().compareTo(o2.getKey()) : ret;
+ }
+ });
+ sortedStatsSet.addAll(statistics.entrySet());
+ for (Long cuboid : cuboids) {
if (statistics.get(cuboid) == null) {
// Get estimate row count for mandatory cuboid
- long tmpRowCount = -1;
for (Map.Entry<Long, Long> entry : sortedStatsSet) {
if (isDescendant(cuboid, entry.getKey())) {
- tmpRowCount = entry.getValue();
+ result.put(cuboid, entry.getValue());
+ break;
+ }
+ }
+ } else {
+ result.put(cuboid, statistics.get(cuboid));
+ }
+ }
+
+ return result;
+ }
+
+ /**
+ * adjust cuboid row count, make sure parent not less than child
+ */
+ public static Map<Long, Long> adjustCuboidStats(Map<Long, Long> statistics) {
+ Map<Long, Long> ret = Maps.newHashMapWithExpectedSize(statistics.size());
+
+ List<Long> cuboids = Lists.newArrayList(statistics.keySet());
+ Collections.sort(cuboids);
+
+ for (Long cuboid : cuboids) {
+ Long rowCount = statistics.get(cuboid);
+ for (Long childCuboid : ret.keySet()) {
+ if (isDescendant(childCuboid, cuboid)) {
+ Long childRowCount = ret.get(childCuboid);
+ if (rowCount < childRowCount) {
+ rowCount = childRowCount;
+ }
+ }
+ }
+ ret.put(cuboid, rowCount);
+ }
+
+ return ret;
+ }
+
+ public static void adjustCuboidStats(Map<Long, Long> mandatoryCuboidsWithStats, Map<Long, Long> statistics) {
+ List<Long> mandatoryCuboids = Lists.newArrayList(mandatoryCuboidsWithStats.keySet());
+ Collections.sort(mandatoryCuboids);
+
+ List<Long> selectedCuboids = Lists.newArrayList(statistics.keySet());
+ Collections.sort(selectedCuboids);
+
+ for (int i = 0; i < mandatoryCuboids.size(); i++) {
+ Long mCuboid = mandatoryCuboids.get(i);
+ if (statistics.get(mCuboid) != null) {
+ mandatoryCuboidsWithStats.put(mCuboid, statistics.get(mCuboid));
+ continue;
+ }
+ int k = 0;
+ // Make sure mCuboid's row count larger than its children's row count in statistics
+ for (; k < selectedCuboids.size(); k++) {
+ Long sCuboid = selectedCuboids.get(k);
+ if (sCuboid > mCuboid) {
+ break;
+ }
+ if (isDescendant(sCuboid, mCuboid)) {
+ Long childRowCount = statistics.get(sCuboid);
+ if (childRowCount > mandatoryCuboidsWithStats.get(mCuboid)) {
+ mandatoryCuboidsWithStats.put(mCuboid, childRowCount);
+ }
+ }
+ }
+ // Make sure mCuboid's row count larger than its children's row count in mandatoryCuboids
+ for (int j = 0; j < i; j++) {
+ Long cCuboid = mandatoryCuboids.get(j);
+ if (isDescendant(cCuboid, mCuboid)) {
+ Long childRowCount = mandatoryCuboidsWithStats.get(cCuboid);
+ if (childRowCount > mandatoryCuboidsWithStats.get(mCuboid)) {
+ mandatoryCuboidsWithStats.put(mCuboid, childRowCount);
+ }
+ }
+ }
+ // Make sure mCuboid's row count lower than its parents' row count in statistics
+ for (; k < selectedCuboids.size(); k++) {
+ Long sCuboid = selectedCuboids.get(k);
+ if (isDescendant(mCuboid, sCuboid)) {
+ Long parentRowCount = statistics.get(sCuboid);
+ if (parentRowCount < mandatoryCuboidsWithStats.get(mCuboid)) {
+ mandatoryCuboidsWithStats.put(mCuboid, parentRowCount);
}
}
- statistics.put(cuboid, tmpRowCount < 0 ? statistics.get(baseCuboid) : tmpRowCount);
}
}
}
@@ -207,4 +334,9 @@ public class CuboidStatsUtil {
public static boolean isDescendant(long cuboidToCheck, long parentCuboid) {
return (cuboidToCheck & parentCuboid) == cuboidToCheck;
}
+
+ private static double calculateRollupRatio(Pair<Long, Long> rollupStats) {
+ double rollupInputCount = rollupStats.getFirst() + rollupStats.getSecond();
+ return rollupInputCount == 0 ? 0 : 1.0 * rollupStats.getFirst() / rollupInputCount;
+ }
}
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
index ba8833a..1ab96a2 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.kylin.common.util.Pair;
import org.junit.Assert;
import org.junit.Test;
@@ -44,6 +45,9 @@ public class CuboidStatsUtilTest {
* / | /
* 00000100 00000010
* */
+
+ private final static long baseCuboidId = 255L;
+
private Set<Long> generateCuboidSet() {
return Sets.newHashSet(255L, 159L, 239L, 50L, 199L, 6L, 4L, 2L);
}
@@ -53,11 +57,11 @@ public class CuboidStatsUtilTest {
countMap.put(255L, 10000L);
countMap.put(159L, 10000L);
- countMap.put(50L, 10000L);
- countMap.put(199L, 10000L);
- countMap.put(6L, 10000L);
- countMap.put(4L, 10000L);
- countMap.put(2L, 10000L);
+ countMap.put(50L, 800L);
+ countMap.put(199L, 200L);
+ countMap.put(6L, 60L);
+ countMap.put(4L, 40L);
+ countMap.put(2L, 20L);
return countMap;
}
@@ -67,26 +71,46 @@ public class CuboidStatsUtilTest {
long totalHitFrequency = 10000L;
- hitFrequencyMap.put(239L, (long) (totalHitFrequency * 0.5));
+ hitFrequencyMap.put(239L, (long) (totalHitFrequency * 0.2));
hitFrequencyMap.put(50L, (long) (totalHitFrequency * 0.2));
- hitFrequencyMap.put(2L, (long) (totalHitFrequency * 0.25));
+ hitFrequencyMap.put(2L, (long) (totalHitFrequency * 0.24));
hitFrequencyMap.put(178L, (long) (totalHitFrequency * 0.05));
+ hitFrequencyMap.put(187L, (long) (totalHitFrequency * 0.3));
+ hitFrequencyMap.put(0L, (long) (totalHitFrequency * 0.01));
return hitFrequencyMap;
}
- private Map<Long, Map<Long, Long>> simulateRollingUpCount() {
- Map<Long, Map<Long, Long>> rollingUpCountMap = Maps.newLinkedHashMap();
+ private Map<Long, Double> simulateHitProbability(long nCuboids) {
+ Map<Long, Long> hitFrequencyMap = simulateHitFrequency();
+ return CuboidStatsUtil.calculateCuboidHitProbability(hitFrequencyMap.keySet(), hitFrequencyMap, nCuboids,
+ CuboidStats.WEIGHT_FOR_UN_QUERY);
+ }
+
+ private Map<Long, Map<Long, Pair<Long, Long>>> simulateRollingUpCount() {
+ Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountMap = Maps.newLinkedHashMap();
+
+ rollingUpCountMap.put(239L, new HashMap<Long, Pair<Long, Long>>() {
+ {
+ put(255L, new Pair<>(990L, 10L));
+ }
+ });
+
+ rollingUpCountMap.put(178L, new HashMap<Long, Pair<Long, Long>>() {
+ {
+ put(255L, new Pair<>(4999L, 1L));
+ }
+ });
- rollingUpCountMap.put(239L, new HashMap<Long, Long>() {
+ rollingUpCountMap.put(187L, new HashMap<Long, Pair<Long, Long>>() {
{
- put(255L, 4000L);
+ put(251L, new Pair<>(3000L, 1000L));
}
});
- rollingUpCountMap.put(178L, new HashMap<Long, Long>() {
+ rollingUpCountMap.put(0L, new HashMap<Long, Pair<Long, Long>>() {
{
- put(255L, 5000L);
+ put(2L, new Pair<>(19L, 1L));
}
});
@@ -101,25 +125,113 @@ public class CuboidStatsUtilTest {
@Test
public void generateMandatoryCuboidSetTest() {
- Set<Long> mandatoryCuboidSet = CuboidStatsUtil.generateMandatoryCuboidSet(simulateCount(),
- simulateHitFrequency(), simulateRollingUpCount(), 1000L);
- Assert.assertTrue(mandatoryCuboidSet.contains(239L));
- Assert.assertTrue(!mandatoryCuboidSet.contains(178L));
+ Map<Long, Long> srcCuboidSet = CuboidStatsUtil.generateSourceCuboidStats(simulateCount(),
+ simulateHitProbability(baseCuboidId), simulateRollingUpCount());
+
+ Assert.assertTrue(srcCuboidSet.get(239L) == 200L);
+ Assert.assertTrue(srcCuboidSet.get(187L) == 1000L);
+ Assert.assertTrue(srcCuboidSet.get(178L) == 800L);
+
+ Assert.assertTrue(!srcCuboidSet.containsKey(0L));
}
@Test
public void complementRowCountForMandatoryCuboidsTest() {
Map<Long, Long> countMap = simulateCount();
- Set<Long> mandatoryCuboidSet = CuboidStatsUtil.generateMandatoryCuboidSet(countMap, simulateHitFrequency(),
- simulateRollingUpCount(), 1000L);
- for (long mandatoryCuboid : mandatoryCuboidSet) {
+ Map<Long, Long> srcCuboidsStats = CuboidStatsUtil.generateSourceCuboidStats(countMap,
+ simulateHitProbability(baseCuboidId), simulateRollingUpCount());
+ for (long mandatoryCuboid : srcCuboidsStats.keySet()) {
Assert.assertNull(countMap.get(mandatoryCuboid));
}
- CuboidStatsUtil.complementRowCountForMandatoryCuboids(countMap, 255L, mandatoryCuboidSet);
- for (long mandatoryCuboid : mandatoryCuboidSet) {
- Assert.assertNotNull(countMap.get(mandatoryCuboid));
- }
- Assert.assertTrue(countMap.get(239L) == 10000L);
+ Assert.assertTrue(srcCuboidsStats.get(239L) == 200L);
+
+ Map<Long, Long> mandatoryCuboidsWithStats2 = Maps.newHashMap();
+ mandatoryCuboidsWithStats2.put(215L, countMap.get(255L));
+ mandatoryCuboidsWithStats2.put(34L, countMap.get(50L));
+ Assert.assertEquals(mandatoryCuboidsWithStats2,
+ CuboidStatsUtil.complementRowCountForCuboids(countMap, mandatoryCuboidsWithStats2.keySet()));
+ }
+
+ @Test
+ public void testAdjustMandatoryCuboidStats() {
+ Map<Long, Long> statistics = Maps.newHashMap();
+ statistics.put(60160L, 1212L);
+
+ Map<Long, Long> cuboidsWithStats = Maps.newHashMap();
+ cuboidsWithStats.put(65280L, 1423L);
+ cuboidsWithStats.put(63232L, 2584421L);
+ cuboidsWithStats.put(61184L, 132L);
+ cuboidsWithStats.put(57088L, 499L);
+ cuboidsWithStats.put(55040L, 708L);
+ cuboidsWithStats.put(38656L, 36507L);
+
+ Map<Long, Double> cuboidHitProbabilityMap = Maps.newHashMap();
+ cuboidHitProbabilityMap.put(65280L, 0.2);
+ cuboidHitProbabilityMap.put(63232L, 0.16);
+ cuboidHitProbabilityMap.put(61184L, 0.16);
+ cuboidHitProbabilityMap.put(57088L, 0.16);
+ cuboidHitProbabilityMap.put(55040L, 0.16);
+ cuboidHitProbabilityMap.put(38656L, 0.16);
+
+ Map<Long, Long> cuboidsWithStatsExpected = Maps.newHashMap(cuboidsWithStats);
+ cuboidsWithStatsExpected.put(65280L, 2584421L);
+ cuboidsWithStatsExpected.put(57088L, 36507L);
+ cuboidsWithStatsExpected.put(55040L, 36507L);
+ cuboidsWithStatsExpected.put(61184L, 1212L);
+
+ CuboidStatsUtil.adjustCuboidStats(cuboidsWithStats, statistics);
+ Assert.assertEquals(cuboidsWithStatsExpected, cuboidsWithStats);
+ }
+
+ /**
+ * 1111(70) 1111(90)
+ * / \ / \
+ * 1011(90) \ 1111(90) \
+ * | \ | \
+ * 0011(40) 1110(50) ==========> 0011(80) 1110(80)
+ * / \ / \ / \ / \
+ * 0001(20) 0010(80) 0100(60) 0001(20) 0010(80) 0100(60)
+ *
+ *
+ * +
+ *
+ *
+ * / \
+ * / \
+ * / 1001(85)
+ * / \
+ * 0111(70) 1000(100)
+ * */
+ @Test
+ public void testAdjustCuboidStats() {
+ Map<Long, Long> statistics = Maps.newHashMap();
+ statistics.put(1L, 20L);
+ statistics.put(2L, 80L);
+ statistics.put(4L, 60L);
+ statistics.put(3L, 40L);
+ statistics.put(11L, 90L);
+ statistics.put(14L, 50L);
+ statistics.put(15L, 70L);
+
+ Map<Long, Long> cuboidsWithStatsExpected = Maps.newHashMap(statistics);
+ cuboidsWithStatsExpected.put(3L, 80L);
+ cuboidsWithStatsExpected.put(14L, 80L);
+ cuboidsWithStatsExpected.put(15L, 90L);
+
+ statistics = CuboidStatsUtil.adjustCuboidStats(statistics);
+ Assert.assertEquals(cuboidsWithStatsExpected, statistics);
+
+ Map<Long, Long> mandatoryCuboidsWithStats = Maps.newHashMap();
+ mandatoryCuboidsWithStats.put(7L, 70L);
+ mandatoryCuboidsWithStats.put(8L, 100L);
+ mandatoryCuboidsWithStats.put(9L, 85L);
+ CuboidStatsUtil.adjustCuboidStats(mandatoryCuboidsWithStats, statistics);
+
+ Map<Long, Long> mandatoryCuboidsWithStatsExpected = Maps.newHashMap();
+ mandatoryCuboidsWithStatsExpected.put(7L, 80L);
+ mandatoryCuboidsWithStatsExpected.put(8L, 80L);
+ mandatoryCuboidsWithStatsExpected.put(9L, 85L);
+ Assert.assertEquals(mandatoryCuboidsWithStatsExpected, mandatoryCuboidsWithStats);
}
@Test
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
index 05458b6..2bffe86 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
@@ -22,9 +22,11 @@ import java.io.IOException;
import java.util.Map;
import java.util.Set;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.cuboid.algorithm.CuboidRecommender;
import org.apache.kylin.cube.cuboid.algorithm.CuboidStats;
import org.slf4j.Logger;
@@ -65,23 +67,45 @@ public class CuboidRecommenderUtil {
/** Trigger cube planner phase two for optimization */
public static Map<Long, Long> getRecommendCuboidList(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
- Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
+ Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap) throws IOException {
+ CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+ Set<Long> currentCuboids = cuboidScheduler.getAllCuboidIds();
Pair<Map<Long, Long>, Map<Long, Double>> statsPair = CuboidStatsReaderUtil
- .readCuboidStatsAndSizeFromCube(cube.getCuboidScheduler().getAllCuboidIds(), cube);
+ .readCuboidStatsAndSizeFromCube(currentCuboids, cube);
+ long baseCuboid = cuboidScheduler.getBaseCuboidId();
+ if (statsPair.getFirst().get(baseCuboid) == null || statsPair.getFirst().get(baseCuboid) == 0L) {
+ logger.info("Base cuboid count in cuboid statistics is 0.");
+ return null;
+ }
+ KylinConfig config = cube.getConfig();
String key = cube.getName();
- long baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
- CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, statsPair.getFirst(), statsPair.getSecond())
- .setHitFrequencyMap(hitFrequencyMap).setRollingUpCountSourceMap(rollingUpCountSourceMap,
- cube.getConfig().getCubePlannerMandatoryRollUpThreshold())
+ double queryUncertaintyRatio = config.getCubePlannerQueryUncertaintyRatio();
+ double bpusMinBenefitRatio = config.getCubePlannerBPUSMinBenefitRatio();
+ CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, statsPair.getFirst(),
+ statsPair.getSecond()) {
+ @Override
+ public Map<Long, Double> estimateCuboidsSize(Map<Long, Long> statistics) {
+ try {
+ return CuboidStatsReaderUtil.readCuboidSizeFromCube(statistics, cube);
+ } catch (IOException e) {
+ logger.warn("Fail to get cuboid size from cube due to ", e);
+ return null;
+ }
+ }
+ }.setQueryUncertaintyRatio(queryUncertaintyRatio) //
+ .setBPUSMinBenefitRatio(bpusMinBenefitRatio) //
+ .setHitFrequencyMap(hitFrequencyMap) //
+ .setRollingUpCountSourceMap(rollingUpCountSourceMap) //
.build();
- return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, cube.getConfig());
+ return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, config);
}
/** For future segment level recommend */
public static Map<Long, Long> getRecommendCuboidList(CubeSegment segment, Map<Long, Long> hitFrequencyMap,
- Map<Long, Map<Long, Long>> rollingUpCountSourceMap, boolean ifForceRecommend) throws IOException {
+ Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap, boolean ifForceRecommend)
+ throws IOException {
if (segment == null) {
return null;
}
@@ -103,9 +127,7 @@ public class CuboidRecommenderUtil {
String key = cube.getName() + "-" + segment.getName();
CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, cubeStatsReader.getCuboidRowEstimatesHLL(),
cubeStatsReader.getCuboidSizeMap()).setHitFrequencyMap(hitFrequencyMap)
- .setRollingUpCountSourceMap(rollingUpCountSourceMap,
- segment.getConfig().getCubePlannerMandatoryRollUpThreshold())
- .build();
+ .setRollingUpCountSourceMap(rollingUpCountSourceMap).build();
return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, segment.getConfig(),
ifForceRecommend);
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
index 1542aa2..ee615c3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
@@ -68,6 +68,29 @@ public class CuboidStatsReaderUtil {
return statisticsMerged.isEmpty() ? null : statisticsMerged;
}
+ public static Map<Long, Double> readCuboidSizeFromCube(Map<Long, Long> statistics, CubeInstance cube)
+ throws IOException {
+ List<CubeSegment> segmentList = cube.getSegments(SegmentStatusEnum.READY);
+ Map<Long, Double> sizeMerged = Maps.newHashMapWithExpectedSize(statistics.size());
+ for (CubeSegment pSegment : segmentList) {
+ CubeStatsReader pReader = new CubeStatsReader(pSegment, null, pSegment.getConfig());
+ Map<Long, Double> pSizeMap = CubeStatsReader.getCuboidSizeMapFromRowCount(pSegment, statistics,
+ pReader.sourceRowCount);
+ for (Long pCuboid : statistics.keySet()) {
+ Double pSize = sizeMerged.get(pCuboid);
+ sizeMerged.put(pCuboid, pSize == null ? pSizeMap.get(pCuboid) : pSize + pSizeMap.get(pCuboid));
+ }
+ }
+ int nSegment = segmentList.size();
+ if (nSegment <= 1) {
+ return sizeMerged;
+ }
+ for (Long pCuboid : statistics.keySet()) {
+ sizeMerged.put(pCuboid, sizeMerged.get(pCuboid) / nSegment);
+ }
+ return sizeMerged;
+ }
+
private static void readCuboidStatsFromSegments(Set<Long> cuboidSet, List<CubeSegment> segmentList,
final Map<Long, Long> statisticsMerged, final Map<Long, Double> sizeMerged) throws IOException {
if (segmentList == null || segmentList.isEmpty()) {
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 11753ec..08b03a4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -32,6 +32,7 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -901,7 +902,8 @@ public class CubeController extends BasicController {
private Map<Long, Long> getRecommendCuboidList(CubeInstance cube) throws IOException {
// Get cuboid source info
Map<Long, Long> optimizeHitFrequencyMap = getSourceCuboidHitFrequency(cube.getName());
- Map<Long, Map<Long, Long>> rollingUpCountSourceMap = cubeService.getCuboidRollingUpStats(cube.getName());
+ Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap = cubeService
+ .getCuboidRollingUpStats(cube.getName());
return cubeService.getRecommendCuboidStatistics(cube, optimizeHitFrequencyMap, rollingUpCountSourceMap);
}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 4944b58..a9fbb97 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -32,6 +32,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -893,7 +894,7 @@ public class CubeService extends BasicService implements InitializingBean {
/** cube planner services */
public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
- Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
+ Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap) throws IOException {
aclEvaluate.checkProjectAdminPermission(cube.getProject());
return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap);
}
@@ -906,12 +907,16 @@ public class CubeService extends BasicService implements InitializingBean {
return formattedQueryCount;
}
- public Map<Long, Map<Long, Long>> formatRollingUpStats(List<List<String>> orgRollingUpCount) {
- Map<Long, Map<Long, Long>> formattedRollingUpStats = Maps.newLinkedHashMap();
+ public Map<Long, Map<Long, Pair<Long, Long>>> formatRollingUpStats(List<List<String>> orgRollingUpCount) {
+ Map<Long, Map<Long, Pair<Long, Long>>> formattedRollingUpStats = Maps.newLinkedHashMap();
for (List<String> rollingUp : orgRollingUpCount) {
- Map<Long, Long> childMap = Maps.newLinkedHashMap();
- childMap.put(Long.parseLong(rollingUp.get(1)), (long) Double.parseDouble(rollingUp.get(2)));
- formattedRollingUpStats.put(Long.parseLong(rollingUp.get(0)), childMap);
+ Map<Long, Pair<Long, Long>> childMap = Maps.newLinkedHashMap();
+ Long srcCuboid = Long.parseLong(rollingUp.get(0));
+ Long tgtCuboid = Long.parseLong(rollingUp.get(1));
+ Long rollupCount = (long) Double.parseDouble(rollingUp.get(2));
+ Long returnCount = (long) Double.parseDouble(rollingUp.get(3));
+ childMap.put(tgtCuboid, new Pair<>(rollupCount, returnCount));
+ formattedRollingUpStats.put(srcCuboid, childMap);
}
return formattedRollingUpStats;
}
@@ -929,15 +934,16 @@ public class CubeService extends BasicService implements InitializingBean {
return formatQueryCount(orgHitFrequency);
}
- public Map<Long, Map<Long, Long>> getCuboidRollingUpStats(String cubeName) {
+ public Map<Long, Map<Long, Pair<Long, Long>>> getCuboidRollingUpStats(String cubeName) {
String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
- String cuboidTarget = QueryCubePropertyEnum.CUBOID_TARGET.toString();
+ String cuboidTgt = QueryCubePropertyEnum.CUBOID_TARGET.toString();
String aggCount = QueryCubePropertyEnum.AGGR_COUNT.toString();
+ String returnCount = QueryCubePropertyEnum.RETURN_COUNT.toString();
String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
- String sql = "select " + cuboidSource + ", " + cuboidTarget + ", avg(" + aggCount + ")" //
+ String sql = "select " + cuboidSource + ", " + cuboidTgt + ", avg(" + aggCount + "), avg(" + returnCount + ")"//
+ " from " + table //
+ " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " //
- + " group by " + cuboidSource + ", " + cuboidTarget;
+ + " group by " + cuboidSource + ", " + cuboidTgt;
List<List<String>> orgRollingUpCount = queryService.querySystemCube(sql).getResults();
return formatRollingUpStats(orgRollingUpCount);
}
[kylin] 01/03: KYLIN-3540 move queryService of CubeController to
CubeService
Posted by nj...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8cfe32cf4f61d218439e57e134ccc3413aa98f89
Author: kyotoYaho <nj...@apache.org>
AuthorDate: Wed Dec 19 10:29:38 2018 +0800
KYLIN-3540 move queryService of CubeController to CubeService
---
.../kylin/rest/controller/CubeController.java | 71 ++---------------
.../org/apache/kylin/rest/service/CubeService.java | 88 ++++++++++++++++++----
2 files changed, 78 insertions(+), 81 deletions(-)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index dcaa259..11753ec 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -56,8 +56,6 @@ import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentRange.TSRange;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metrics.MetricsManager;
-import org.apache.kylin.metrics.property.QueryCubePropertyEnum;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -70,7 +68,6 @@ import org.apache.kylin.rest.request.JobBuildRequest;
import org.apache.kylin.rest.request.JobBuildRequest2;
import org.apache.kylin.rest.request.JobOptimizeRequest;
import org.apache.kylin.rest.request.LookupSnapshotBuildRequest;
-import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.EnvelopeResponse;
@@ -80,7 +77,6 @@ import org.apache.kylin.rest.response.ResponseCode;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
import org.apache.kylin.rest.service.ProjectService;
-import org.apache.kylin.rest.service.QueryService;
import org.apache.kylin.rest.util.ValidateUtil;
import org.apache.kylin.source.kafka.util.KafkaClient;
import org.slf4j.Logger;
@@ -124,10 +120,6 @@ public class CubeController extends BasicController {
@Qualifier("projectService")
private ProjectService projectService;
- @Autowired
- @Qualifier("queryService")
- private QueryService queryService;
-
@RequestMapping(value = "/validate/{cubeName}", method = RequestMethod.GET, produces = { "application/json" })
@ResponseBody
public EnvelopeResponse<Boolean> validateModelName(@PathVariable String cubeName) {
@@ -873,7 +865,7 @@ public class CubeController extends BasicController {
Map<Long, Long> queryMatchMap = null;
try {
hitFrequencyMap = getTargetCuboidHitFrequency(cubeName);
- queryMatchMap = getCuboidQueryMatchCount(cubeName);
+ queryMatchMap = cubeService.getCuboidQueryMatchCount(cubeName);
} catch (Exception e) {
logger.warn("Fail to query on system cube due to " + e);
}
@@ -899,7 +891,7 @@ public class CubeController extends BasicController {
// Get cuboid target info for displaying heat map of cuboid hit
Map<Long, Long> displayHitFrequencyMap = getTargetCuboidHitFrequency(cubeName);
// Get exactly matched cuboid query count
- Map<Long, Long> queryMatchMap = getCuboidQueryMatchCount(cubeName);
+ Map<Long, Long> queryMatchMap = cubeService.getCuboidQueryMatchCount(cubeName);
Set<Long> currentCuboidSet = cube.getCuboidScheduler().getAllCuboidIds();
return cubeService.getCuboidTreeResponse(cuboidScheduler, recommendCuboidStatsMap, displayHitFrequencyMap,
@@ -909,69 +901,16 @@ public class CubeController extends BasicController {
private Map<Long, Long> getRecommendCuboidList(CubeInstance cube) throws IOException {
// Get cuboid source info
Map<Long, Long> optimizeHitFrequencyMap = getSourceCuboidHitFrequency(cube.getName());
- Map<Long, Map<Long, Long>> rollingUpCountSourceMap = getCuboidRollingUpCount(cube.getName());
+ Map<Long, Map<Long, Long>> rollingUpCountSourceMap = cubeService.getCuboidRollingUpStats(cube.getName());
return cubeService.getRecommendCuboidStatistics(cube, optimizeHitFrequencyMap, rollingUpCountSourceMap);
}
private Map<Long, Long> getSourceCuboidHitFrequency(String cubeName) {
- return getCuboidHitFrequency(cubeName, true);
+ return cubeService.getCuboidHitFrequency(cubeName, true);
}
private Map<Long, Long> getTargetCuboidHitFrequency(String cubeName) {
- return getCuboidHitFrequency(cubeName, false);
- }
-
- private Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean isCuboidSource) {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
- String cuboidColumn = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
- if (!isCuboidSource) {
- cuboidColumn = QueryCubePropertyEnum.CUBOID_TARGET.toString();
- }
- String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
- String table = cubeService.getMetricsManager()
- .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
- String sql = "select " + cuboidColumn + ", sum(" + hitMeasure + ") " //
- + "from " + table//
- + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " //
- + "group by " + cuboidColumn;
- sqlRequest.setSql(sql);
- List<List<String>> orgHitFrequency = queryService.doQueryWithCache(sqlRequest).getResults();
- return cubeService.formatQueryCount(orgHitFrequency);
- }
-
- private Map<Long, Map<Long, Long>> getCuboidRollingUpCount(String cubeName) {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
- String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
- String cuboidTarget = QueryCubePropertyEnum.CUBOID_TARGET.toString();
- String aggCount = QueryCubePropertyEnum.AGGR_COUNT.toString();
- String table = cubeService.getMetricsManager()
- .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
- String sql = "select " + cuboidSource + ", " + cuboidTarget + ", sum(" + aggCount + ")/count(*) " //
- + "from " + table //
- + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " //
- + "group by " + cuboidSource + ", " + cuboidTarget;
- sqlRequest.setSql(sql);
- List<List<String>> orgRollingUpCount = queryService.doQueryWithCache(sqlRequest).getResults();
- return cubeService.formatRollingUpCount(orgRollingUpCount);
- }
-
- private Map<Long, Long> getCuboidQueryMatchCount(String cubeName) {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
- String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
- String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
- String table = cubeService.getMetricsManager()
- .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
- String sql = "select " + cuboidSource + ", sum(" + hitMeasure + ") " //
- + "from " + table //
- + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' and "
- + QueryCubePropertyEnum.IF_MATCH.toString() + " = true " //
- + "group by " + cuboidSource;
- sqlRequest.setSql(sql);
- List<List<String>> orgMatchHitFrequency = queryService.doQueryWithCache(sqlRequest).getResults();
- return cubeService.formatQueryCount(orgMatchHitFrequency);
+ return cubeService.getCuboidHitFrequency(cubeName, false);
}
/**
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index e541c37..e24e9de 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -65,6 +65,8 @@ import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.metrics.MetricsManager;
+import org.apache.kylin.metrics.property.QueryCubePropertyEnum;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
@@ -72,6 +74,7 @@ import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
import org.apache.kylin.rest.request.MetricsRequest;
+import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse.NodeInfo;
@@ -121,6 +124,10 @@ public class CubeService extends BasicService implements InitializingBean {
private ModelService modelService;
@Autowired
+ @Qualifier("queryService")
+ private QueryService queryService;
+
+ @Autowired
private AclEvaluate aclEvaluate;
@Autowired
@@ -472,7 +479,8 @@ public class CubeService extends BasicService implements InitializingBean {
hr = new HBaseResponse();
CubeInstance cube = CubeManager.getInstance(getConfig()).getCube(cubeName);
- if (cube.getStorageType() == IStorageAware.ID_HBASE || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE) {
+ if (cube.getStorageType() == IStorageAware.ID_HBASE
+ || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE) {
try {
logger.debug("Loading HTable info " + cubeName + ", " + tableName);
@@ -547,9 +555,9 @@ public class CubeService extends BasicService implements InitializingBean {
}
public boolean isOrphonSegment(CubeInstance cube, String segId) {
- List<JobInstance> jobInstances = jobService.searchJobsByCubeName(cube.getName(),
- cube.getProject(), Lists.newArrayList(JobStatusEnum.NEW, JobStatusEnum.PENDING, JobStatusEnum.RUNNING,
- JobStatusEnum.ERROR, JobStatusEnum.STOPPED),
+ List<JobInstance> jobInstances = jobService.searchJobsByCubeName(
+ cube.getName(), cube.getProject(), Lists.newArrayList(JobStatusEnum.NEW, JobStatusEnum.PENDING,
+ JobStatusEnum.RUNNING, JobStatusEnum.ERROR, JobStatusEnum.STOPPED),
JobTimeFilterEnum.ALL, JobService.JobSearchMode.CUBING_ONLY);
for (JobInstance jobInstance : jobInstances) {
// if there are segment related jobs, can not delete this segment.
@@ -886,6 +894,12 @@ public class CubeService extends BasicService implements InitializingBean {
}
/** cube planner services */
+ public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
+ Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
+ aclEvaluate.checkProjectAdminPermission(cube.getProject());
+ return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap);
+ }
+
public Map<Long, Long> formatQueryCount(List<List<String>> orgQueryCount) {
Map<Long, Long> formattedQueryCount = Maps.newLinkedHashMap();
for (List<String> hit : orgQueryCount) {
@@ -894,20 +908,64 @@ public class CubeService extends BasicService implements InitializingBean {
return formattedQueryCount;
}
- public Map<Long, Map<Long, Long>> formatRollingUpCount(List<List<String>> orgRollingUpCount) {
- Map<Long, Map<Long, Long>> formattedRollingUpCount = Maps.newLinkedHashMap();
+ public Map<Long, Map<Long, Long>> formatRollingUpStats(List<List<String>> orgRollingUpCount) {
+ Map<Long, Map<Long, Long>> formattedRollingUpStats = Maps.newLinkedHashMap();
for (List<String> rollingUp : orgRollingUpCount) {
Map<Long, Long> childMap = Maps.newLinkedHashMap();
childMap.put(Long.parseLong(rollingUp.get(1)), (long) Double.parseDouble(rollingUp.get(2)));
- formattedRollingUpCount.put(Long.parseLong(rollingUp.get(0)), childMap);
- }
- return formattedRollingUpCount;
- }
-
- public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
- Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
- aclEvaluate.checkProjectAdminPermission(cube.getProject());
- return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap);
+ formattedRollingUpStats.put(Long.parseLong(rollingUp.get(0)), childMap);
+ }
+ return formattedRollingUpStats;
+ }
+
+ public Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean isCuboidSource) {
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ String cuboidColumn = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
+ if (!isCuboidSource) {
+ cuboidColumn = QueryCubePropertyEnum.CUBOID_TARGET.toString();
+ }
+ String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
+ String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
+ String sql = "select " + cuboidColumn + ", sum(" + hitMeasure + ")" //
+ + " from " + table//
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " //
+ + " group by " + cuboidColumn;
+ sqlRequest.setSql(sql);
+ List<List<String>> orgHitFrequency = queryService.doQueryWithCache(sqlRequest).getResults();
+ return formatQueryCount(orgHitFrequency);
+ }
+
+ public Map<Long, Map<Long, Long>> getCuboidRollingUpStats(String cubeName) {
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
+ String cuboidTarget = QueryCubePropertyEnum.CUBOID_TARGET.toString();
+ String aggCount = QueryCubePropertyEnum.AGGR_COUNT.toString();
+ String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
+ String sql = "select " + cuboidSource + ", " + cuboidTarget + ", sum(" + aggCount + ")/count(*)" //
+ + " from " + table //
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " //
+ + " group by " + cuboidSource + ", " + cuboidTarget;
+ sqlRequest.setSql(sql);
+ List<List<String>> orgRollingUpCount = queryService.doQueryWithCache(sqlRequest).getResults();
+ return formatRollingUpStats(orgRollingUpCount);
+ }
+
+ public Map<Long, Long> getCuboidQueryMatchCount(String cubeName) {
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
+ String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
+ String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
+ String sql = "select " + cuboidSource + ", sum(" + hitMeasure + ")" //
+ + " from " + table //
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "'" //
+ + " and " + QueryCubePropertyEnum.IF_MATCH.toString() + " = true" //
+ + " group by " + cuboidSource;
+ sqlRequest.setSql(sql);
+ List<List<String>> orgMatchHitFrequency = queryService.doQueryWithCache(sqlRequest).getResults();
+ return formatQueryCount(orgMatchHitFrequency);
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
[kylin] 02/03: KYLIN-3540 refactor the interface of querying on
SYSTEM project
Posted by nj...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4db6a37c7220c122cedb9fac1a2c735f10e27226
Author: kyotoYaho <nj...@apache.org>
AuthorDate: Wed Dec 19 10:40:33 2018 +0800
KYLIN-3540 refactor the interface of querying on SYSTEM project
---
.../kylin/rest/controller/DashboardController.java | 66 ++++++++++++----------
.../org/apache/kylin/rest/service/CubeService.java | 27 +++------
.../kylin/rest/service/DashboardService.java | 34 ++++++-----
.../apache/kylin/rest/service/QueryService.java | 14 ++++-
4 files changed, 73 insertions(+), 68 deletions(-)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
index ee9fdcd..846d6d3 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
@@ -16,15 +16,12 @@
* limitations under the License.
*/
-
package org.apache.kylin.rest.controller;
import java.util.List;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metrics.MetricsManager;
-import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.service.CubeService;
@@ -57,62 +54,69 @@ public class DashboardController extends BasicController {
@RequestMapping(value = "/metric/cube", method = { RequestMethod.GET })
@ResponseBody
- public MetricsResponse getCubeMetrics(@RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "cubeName", required = false) String cubeName) {
+ public MetricsResponse getCubeMetrics(@RequestParam(value = "projectName", required = false) String projectName,
+ @RequestParam(value = "cubeName", required = false) String cubeName) {
checkAuthorization(projectName);
return dashboardService.getCubeMetrics(projectName, cubeName);
}
@RequestMapping(value = "/metric/query", method = RequestMethod.GET)
@ResponseBody
- public MetricsResponse getQueryMetrics(@RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
+ public MetricsResponse getQueryMetrics(@RequestParam(value = "projectName", required = false) String projectName,
+ @RequestParam(value = "cubeName", required = false) String cubeName,
+ @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
checkAuthorization(projectName);
MetricsResponse queryMetrics = new MetricsResponse();
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
String sql = dashboardService.getQueryMetricsSQL(startTime, endTime, projectName, cubeName);
- sqlRequest.setSql(sql);
- SQLResponse sqlResponse = queryService.doQueryWithCache(sqlRequest);
- if(!sqlResponse.getIsException()){
- queryMetrics.increase("queryCount", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
- queryMetrics.increase("avgQueryLatency", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
- queryMetrics.increase("maxQueryLatency", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
- queryMetrics.increase("minQueryLatency", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
+ SQLResponse sqlResponse = queryService.querySystemCube(sql);
+ if (!sqlResponse.getIsException()) {
+ queryMetrics.increase("queryCount",
+ dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
+ queryMetrics.increase("avgQueryLatency",
+ dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
+ queryMetrics.increase("maxQueryLatency",
+ dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
+ queryMetrics.increase("minQueryLatency",
+ dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
}
return queryMetrics;
}
@RequestMapping(value = "/metric/job", method = RequestMethod.GET)
@ResponseBody
- public MetricsResponse getJobMetrics(@RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
+ public MetricsResponse getJobMetrics(@RequestParam(value = "projectName", required = false) String projectName,
+ @RequestParam(value = "cubeName", required = false) String cubeName,
+ @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
checkAuthorization(projectName);
MetricsResponse jobMetrics = new MetricsResponse();
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
String sql = dashboardService.getJobMetricsSQL(startTime, endTime, projectName, cubeName);
- sqlRequest.setSql(sql);
- SQLResponse sqlResponse = queryService.doQueryWithCache(sqlRequest);
- if(!sqlResponse.getIsException()){
+ SQLResponse sqlResponse = queryService.querySystemCube(sql);
+ if (!sqlResponse.getIsException()) {
jobMetrics.increase("jobCount", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
- jobMetrics.increase("avgJobBuildTime", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
- jobMetrics.increase("maxJobBuildTime", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
- jobMetrics.increase("minJobBuildTime", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
+ jobMetrics.increase("avgJobBuildTime",
+ dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
+ jobMetrics.increase("maxJobBuildTime",
+ dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
+ jobMetrics.increase("minJobBuildTime",
+ dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
}
return jobMetrics;
}
@RequestMapping(value = "/chart/{category}/{metric}/{dimension}", method = RequestMethod.GET)
@ResponseBody
- public MetricsResponse getChartData(@PathVariable String dimension, @PathVariable String metric, @PathVariable String category, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
+ public MetricsResponse getChartData(@PathVariable String dimension, @PathVariable String metric,
+ @PathVariable String category, @RequestParam(value = "projectName", required = false) String projectName,
+ @RequestParam(value = "cubeName", required = false) String cubeName,
+ @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
checkAuthorization(projectName);
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
- String sql = dashboardService.getChartSQL(startTime, endTime, projectName, cubeName, dimension, metric, category);
- sqlRequest.setSql(sql);
- return dashboardService.transformChartData(queryService.doQueryWithCache(sqlRequest));
+ String sql = dashboardService.getChartSQL(startTime, endTime, projectName, cubeName, dimension, metric,
+ category);
+ return dashboardService.transformChartData(queryService.querySystemCube(sql));
}
- private void checkAuthorization(String projectName){
- if (projectName!=null && !projectName.isEmpty()) {
+ private void checkAuthorization(String projectName) {
+ if (projectName != null && !projectName.isEmpty()) {
ProjectInstance project = dashboardService.getProjectManager().getProject(projectName);
try {
dashboardService.checkAuthorization(project);
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index e24e9de..4944b58 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -65,7 +65,6 @@ import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.metrics.MetricsManager;
import org.apache.kylin.metrics.property.QueryCubePropertyEnum;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.BadRequestException;
@@ -74,7 +73,6 @@ import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
import org.apache.kylin.rest.request.MetricsRequest;
-import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse.NodeInfo;
@@ -919,42 +917,32 @@ public class CubeService extends BasicService implements InitializingBean {
}
public Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean isCuboidSource) {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
- String cuboidColumn = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
- if (!isCuboidSource) {
- cuboidColumn = QueryCubePropertyEnum.CUBOID_TARGET.toString();
- }
+ String cuboidColumn = isCuboidSource ? QueryCubePropertyEnum.CUBOID_SOURCE.toString()
+ : QueryCubePropertyEnum.CUBOID_TARGET.toString();
String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
String sql = "select " + cuboidColumn + ", sum(" + hitMeasure + ")" //
+ " from " + table//
- + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " //
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "'" //
+ " group by " + cuboidColumn;
- sqlRequest.setSql(sql);
- List<List<String>> orgHitFrequency = queryService.doQueryWithCache(sqlRequest).getResults();
+ List<List<String>> orgHitFrequency = queryService.querySystemCube(sql).getResults();
return formatQueryCount(orgHitFrequency);
}
public Map<Long, Map<Long, Long>> getCuboidRollingUpStats(String cubeName) {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
String cuboidTarget = QueryCubePropertyEnum.CUBOID_TARGET.toString();
String aggCount = QueryCubePropertyEnum.AGGR_COUNT.toString();
String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
- String sql = "select " + cuboidSource + ", " + cuboidTarget + ", sum(" + aggCount + ")/count(*)" //
+ String sql = "select " + cuboidSource + ", " + cuboidTarget + ", avg(" + aggCount + ")" //
+ " from " + table //
+ " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " //
+ " group by " + cuboidSource + ", " + cuboidTarget;
- sqlRequest.setSql(sql);
- List<List<String>> orgRollingUpCount = queryService.doQueryWithCache(sqlRequest).getResults();
+ List<List<String>> orgRollingUpCount = queryService.querySystemCube(sql).getResults();
return formatRollingUpStats(orgRollingUpCount);
}
public Map<Long, Long> getCuboidQueryMatchCount(String cubeName) {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
@@ -963,8 +951,7 @@ public class CubeService extends BasicService implements InitializingBean {
+ " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "'" //
+ " and " + QueryCubePropertyEnum.IF_MATCH.toString() + " = true" //
+ " group by " + cuboidSource;
- sqlRequest.setSql(sql);
- List<List<String>> orgMatchHitFrequency = queryService.doQueryWithCache(sqlRequest).getResults();
+ List<List<String>> orgMatchHitFrequency = queryService.querySystemCube(sql).getResults();
return formatQueryCount(orgMatchHitFrequency);
}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java b/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
index ec395e0..3910245 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
@@ -58,9 +58,12 @@ public class DashboardService extends BasicService {
}
private enum QueryDimensionEnum {
- PROJECT(QueryPropertyEnum.PROJECT.toString()), CUBE(QueryPropertyEnum.REALIZATION.toString()), DAY(
- TimePropertyEnum.DAY_DATE.toString()), WEEK(
- TimePropertyEnum.WEEK_BEGIN_DATE.toString()), MONTH(TimePropertyEnum.MONTH.toString());
+ PROJECT(QueryPropertyEnum.PROJECT.toString()), //
+ CUBE(QueryPropertyEnum.REALIZATION.toString()), //
+ DAY(TimePropertyEnum.DAY_DATE.toString()), //
+ WEEK(TimePropertyEnum.WEEK_BEGIN_DATE.toString()), //
+ MONTH(TimePropertyEnum.MONTH.toString());
+
private final String sql;
QueryDimensionEnum(String sql) {
@@ -73,9 +76,12 @@ public class DashboardService extends BasicService {
};
private enum JobDimensionEnum {
- PROJECT(JobPropertyEnum.PROJECT.toString()), CUBE(JobPropertyEnum.CUBE.toString()), DAY(
- TimePropertyEnum.DAY_DATE.toString()), WEEK(
- TimePropertyEnum.WEEK_BEGIN_DATE.toString()), MONTH(TimePropertyEnum.MONTH.toString());
+ PROJECT(JobPropertyEnum.PROJECT.toString()), //
+ CUBE(JobPropertyEnum.CUBE.toString()), //
+ DAY(TimePropertyEnum.DAY_DATE.toString()), //
+ WEEK(TimePropertyEnum.WEEK_BEGIN_DATE.toString()), //
+ MONTH(TimePropertyEnum.MONTH.toString());
+
private final String sql;
JobDimensionEnum(String sql) {
@@ -88,10 +94,10 @@ public class DashboardService extends BasicService {
};
private enum QueryMetricEnum {
- QUERY_COUNT("count(*)"), AVG_QUERY_LATENCY("sum(" + QueryPropertyEnum.TIME_COST.toString() + ")/(count("
- + QueryPropertyEnum.TIME_COST.toString() + "))"), MAX_QUERY_LATENCY(
- "max(" + QueryPropertyEnum.TIME_COST.toString() + ")"), MIN_QUERY_LATENCY(
- "min(" + QueryPropertyEnum.TIME_COST.toString() + ")");
+ QUERY_COUNT("count(*)"), //
+ AVG_QUERY_LATENCY("avg(" + QueryPropertyEnum.TIME_COST.toString() + ")"), //
+ MAX_QUERY_LATENCY("max(" + QueryPropertyEnum.TIME_COST.toString() + ")"), //
+ MIN_QUERY_LATENCY("min(" + QueryPropertyEnum.TIME_COST.toString() + ")");
private final String sql;
@@ -105,10 +111,10 @@ public class DashboardService extends BasicService {
}
private enum JobMetricEnum {
- JOB_COUNT("count(*)"), AVG_JOB_BUILD_TIME("sum(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")/count("
- + JobPropertyEnum.PER_BYTES_TIME_COST + ")"), MAX_JOB_BUILD_TIME(
- "max(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"), MIN_JOB_BUILD_TIME(
- "min(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")");
+ JOB_COUNT("count(*)"), //
+ AVG_JOB_BUILD_TIME("avg(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"), //
+ MAX_JOB_BUILD_TIME("max(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"), //
+ MIN_JOB_BUILD_TIME("min(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")");
private final String sql;
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 02bb804..da4fcb6 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -93,6 +93,7 @@ import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.metadata.querymeta.TableMeta;
import org.apache.kylin.metadata.querymeta.TableMetaWithType;
import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metrics.MetricsManager;
import org.apache.kylin.query.QueryConnection;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.util.PushDownUtil;
@@ -352,6 +353,13 @@ public class QueryService extends BasicService {
logger.info(stringBuilder.toString());
}
+ public SQLResponse querySystemCube(String sql) {
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ sqlRequest.setSql(sql);
+ return doQueryWithCache(sqlRequest, false);
+ }
+
public SQLResponse doQueryWithCache(SQLRequest sqlRequest) {
long t = System.currentTimeMillis();
aclEvaluate.checkProjectReadPermission(sqlRequest.getProject());
@@ -375,7 +383,8 @@ public class QueryService extends BasicService {
// project not found
ProjectManager mgr = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
if (mgr.getProject(sqlRequest.getProject()) == null) {
- throw new BadRequestException(String.format(Locale.ROOT, msg.getPROJECT_NOT_FOUND(), sqlRequest.getProject()));
+ throw new BadRequestException(
+ String.format(Locale.ROOT, msg.getPROJECT_NOT_FOUND(), sqlRequest.getProject()));
}
if (StringUtils.isBlank(sqlRequest.getSql())) {
throw new BadRequestException(msg.getNULL_EMPTY_SQL());
@@ -1020,8 +1029,7 @@ public class QueryService extends BasicService {
}
private SQLResponse getPrepareOnlySqlResponse(String projectName, String correctedSql, Connection conn,
- Boolean isPushDown,
- List<List<String>> results, List<SelectedColumnMeta> columnMetas) throws SQLException {
+ Boolean isPushDown, List<List<String>> results, List<SelectedColumnMeta> columnMetas) throws SQLException {
CalcitePrepareImpl.KYLIN_ONLY_PREPARE.set(true);