You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@kylin.apache.org by GitBox <gi...@apache.org> on 2018/12/25 01:28:59 UTC
[GitHub] kyotoYaho closed pull request #407: KYLIN-3540 estimate the row
counts of source cuboids which are not built & remove mandatory cuboids
recommendation
kyotoYaho closed pull request #407: KYLIN-3540 estimate the row counts of source cuboids which are not built & remove mandatory cuboids recommendation
URL: https://github.com/apache/kylin/pull/407
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f67f6b3479..b63062e31a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -316,8 +316,7 @@ public String getMetastoreBigCellHdfsDirectory() {
public String getReadHdfsWorkingDirectory() {
if (StringUtils.isNotEmpty(getHBaseClusterFs())) {
Path workingDir = new Path(getHdfsWorkingDirectory());
- return new Path(getHBaseClusterFs(), Path.getPathWithoutSchemeAndAuthority(workingDir)).toString()
- + "/";
+ return new Path(getHBaseClusterFs(), Path.getPathWithoutSchemeAndAuthority(workingDir)).toString() + "/";
}
return getHdfsWorkingDirectory();
@@ -644,8 +643,12 @@ public int getCubePlannerRecommendCuboidCacheMaxSize() {
return Integer.parseInt(getOptional("kylin.cube.cubeplanner.recommend-cache-max-size", "200"));
}
- public long getCubePlannerMandatoryRollUpThreshold() {
- return Long.parseLong(getOptional("kylin.cube.cubeplanner.mandatory-rollup-threshold", "1000"));
+ public double getCubePlannerQueryUncertaintyRatio() {
+ return Double.parseDouble(getOptional("kylin.cube.cubeplanner.query-uncertainty-ratio", "0.1"));
+ }
+
+ public double getCubePlannerBPUSMinBenefitRatio() {
+ return Double.parseDouble(getOptional("kylin.cube.cubeplanner.bpus-min-benefit-ratio", "0.01"));
}
public int getCubePlannerAgreedyAlgorithmAutoThreshold() {
@@ -1910,12 +1913,13 @@ public boolean isJsonAlwaysSmallCell() {
}
public int getSmallCellMetadataWarningThreshold() {
- return Integer.parseInt(getOptional("kylin.metadata.jdbc.small-cell-meta-size-warning-threshold",
- String.valueOf(100 << 20))); //100mb
+ return Integer.parseInt(
+ getOptional("kylin.metadata.jdbc.small-cell-meta-size-warning-threshold", String.valueOf(100 << 20))); //100mb
}
public int getSmallCellMetadataErrorThreshold() {
- return Integer.parseInt(getOptional("kylin.metadata.jdbc.small-cell-meta-size-error-threshold", String.valueOf(1 << 30))); // 1gb
+ return Integer.parseInt(
+ getOptional("kylin.metadata.jdbc.small-cell-meta-size-error-threshold", String.valueOf(1 << 30))); // 1gb
}
public int getJdbcResourceStoreMaxCellSize() {
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
index 6316858d58..39c52dafe9 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
@@ -142,7 +142,7 @@ public boolean ifEfficient(CuboidBenefitModel best) {
}
public double getMinBenefitRatio() {
- return 0.01;
+ return cuboidStats.getBpusMinBenefitRatio();
}
@Override
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
index baacb51791..0e6a844a95 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
@@ -154,12 +154,11 @@ public static CuboidRecommender getInstance() {
Map<Long, Long> recommendCuboidsWithStats = Maps.newLinkedHashMap();
for (Long cuboid : recommendCuboidList) {
- if (cuboid.equals(cuboidStats.getBaseCuboid())) {
- recommendCuboidsWithStats.put(cuboid, cuboidStats.getCuboidCount(cuboid));
- } else if (cuboidStats.getAllCuboidsForSelection().contains(cuboid)) {
- recommendCuboidsWithStats.put(cuboid, cuboidStats.getCuboidCount(cuboid));
+ if (cuboid == 0L) {
+ // for zero cuboid, just simply recommend the cheapest cuboid.
+ handleCuboidZeroRecommend(cuboidStats, recommendCuboidsWithStats);
} else {
- recommendCuboidsWithStats.put(cuboid, -1L);
+ recommendCuboidsWithStats.put(cuboid, cuboidStats.getCuboidCount(cuboid));
}
}
@@ -168,4 +167,20 @@ public static CuboidRecommender getInstance() {
}
return recommendCuboidsWithStats;
}
+
+ private void handleCuboidZeroRecommend(CuboidStats cuboidStats, Map<Long, Long> recommendCuboidsWithStats) {
+ Map<Long, Long> statistics = cuboidStats.getStatistics();
+ Long cheapestCuboid = null;
+ Long cheapestCuboidCount = Long.MAX_VALUE;
+ for (Map.Entry<Long, Long> cuboidStatsEntry : statistics.entrySet()) {
+ if (cuboidStatsEntry.getValue() < cheapestCuboidCount) {
+ cheapestCuboid = cuboidStatsEntry.getKey();
+ cheapestCuboidCount = cuboidStatsEntry.getValue();
+ }
+ }
+ if (cheapestCuboid != null) {
+ logger.info("recommend cuboid:{} instead of cuboid zero", cheapestCuboid);
+ recommendCuboidsWithStats.put(cheapestCuboid, cheapestCuboidCount);
+ }
+ }
}
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
index 78a6c5b517..c22ab9a00d 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
@@ -18,57 +18,72 @@
package org.apache.kylin.cube.cuboid.algorithm;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.common.util.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
public class CuboidStats {
private static final Logger logger = LoggerFactory.getLogger(CuboidStats.class);
+ static final double WEIGHT_FOR_UN_QUERY = 0.2;
+ static final double BPUS_MIN_BENEFIT_RATIO = 0.001;
+
public static class Builder {
private static final long THRESHOLD_ROLL_UP_FOR_MANDATORY = 1000L;
// Required parameters
private String key;
+ private Long nTotalCuboids;
private Long baseCuboid;
+ private double queryUncertaintyRatio = WEIGHT_FOR_UN_QUERY;
+ private double bpusMinBenefitRatio = BPUS_MIN_BENEFIT_RATIO;
private Map<Long, Long> statistics;
private Map<Long, Double> size;
// Optional parameters - initialized to default values
private Set<Long> mandatoryCuboids = null;
//// These two properties are for generating mandatory cuboids
- private Map<Long, Map<Long, Long>> rollingUpCountSourceMap = null;
- private Long rollUpThresholdForMandatory = null;
+ private Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap = null;
private Map<Long, Long> hitFrequencyMap = null;
private Map<Long, Map<Long, Long>> scanCountSourceMap = null;
public Builder(String key, Long baseCuboid, Map<Long, Long> statistics, Map<Long, Double> size) {
+ this(key, baseCuboid, baseCuboid, statistics, size);
+ }
+
+ public Builder(String key, Long nTotalCuboids, Long baseCuboid, Map<Long, Long> statistics,
+ Map<Long, Double> size) {
this.key = key;
+ this.nTotalCuboids = nTotalCuboids;
this.baseCuboid = baseCuboid;
this.statistics = statistics;
this.size = size;
}
- public Builder setRollingUpCountSourceMap(Map<Long, Map<Long, Long>> rollingUpCountSourceMap) {
- this.rollingUpCountSourceMap = rollingUpCountSourceMap;
- this.rollUpThresholdForMandatory = THRESHOLD_ROLL_UP_FOR_MANDATORY;
+ public Builder setQueryUncertaintyRatio(double queryUncertaintyRatio) {
+ this.queryUncertaintyRatio = queryUncertaintyRatio;
return this;
}
- public Builder setRollingUpCountSourceMap(Map<Long, Map<Long, Long>> rollingUpCountSourceMap,
- long rollUpThresholdForMandatory) {
+ public Builder setBPUSMinBenefitRatio(double bpusMinBenefitRatio) {
+ this.bpusMinBenefitRatio = bpusMinBenefitRatio;
+ return this;
+ }
+
+ public Builder setRollingUpCountSourceMap(Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap) {
this.rollingUpCountSourceMap = rollingUpCountSourceMap;
- this.rollUpThresholdForMandatory = rollUpThresholdForMandatory;
return this;
}
@@ -87,6 +102,10 @@ public Builder setScanCountSourceMap(Map<Long, Map<Long, Long>> scanCountSourceM
return this;
}
+ public Map<Long, Double> estimateCuboidsSize(Map<Long, Long> statistics) {
+ return null;
+ }
+
public CuboidStats build() {
Preconditions.checkNotNull(key, "key should not be null");
Preconditions.checkNotNull(baseCuboid, "baseCuboid should not be null");
@@ -96,23 +115,36 @@ public CuboidStats build() {
"row count should exist for base cuboid " + baseCuboid);
Preconditions.checkState(statistics.keySet().equals(size.keySet()),
"statistics & size should own the same key set");
+ statistics = CuboidStatsUtil.adjustCuboidStats(statistics);
+
+ if (hitFrequencyMap != null && rollingUpCountSourceMap != null) {
+ Map<Long, Double> cuboidHitProbabilityMap = CuboidStatsUtil.calculateCuboidHitProbability(
+ hitFrequencyMap.keySet(), hitFrequencyMap, nTotalCuboids, queryUncertaintyRatio);
+ Map<Long, Long> srcCuboidsStats = CuboidStatsUtil.generateSourceCuboidStats(statistics,
+ cuboidHitProbabilityMap, rollingUpCountSourceMap);
+
+ statistics.putAll(srcCuboidsStats);
+
+ Map<Long, Double> estimatedSize = estimateCuboidsSize(statistics);
+ if (estimatedSize != null && !estimatedSize.isEmpty()) {
+ size = Maps.newHashMap(estimatedSize);
+ }
+ }
+
if (mandatoryCuboids == null) {
mandatoryCuboids = Sets.newHashSet();
- }
- if (rollingUpCountSourceMap != null) {
- mandatoryCuboids.addAll(CuboidStatsUtil.generateMandatoryCuboidSet(statistics, hitFrequencyMap,
- rollingUpCountSourceMap, rollUpThresholdForMandatory));
+ } else if (!mandatoryCuboids.isEmpty()) {
+ statistics.putAll(CuboidStatsUtil.complementRowCountForCuboids(statistics, mandatoryCuboids));
}
- return new CuboidStats(key, baseCuboid, mandatoryCuboids, statistics, size, hitFrequencyMap,
- scanCountSourceMap);
+ return new CuboidStats(key, baseCuboid, queryUncertaintyRatio, bpusMinBenefitRatio, mandatoryCuboids,
+ statistics, size, hitFrequencyMap, scanCountSourceMap);
}
}
- private static final double WEIGHT_FOR_UN_QUERY = 0.2;
-
private String key;
private long baseCuboid;
+ private double bpusMinBenefitRatio;
private ImmutableSet<Long> mandatoryCuboidSet;
private ImmutableSet<Long> selectionCuboidSet;
private ImmutableMap<Long, Long> cuboidCountMap;
@@ -123,11 +155,13 @@ public CuboidStats build() {
private ImmutableMap<Long, List<Long>> directChildrenCache;
private Map<Long, Set<Long>> allDescendantsCache;
- private CuboidStats(String key, long baseCuboidId, Set<Long> mandatoryCuboids, Map<Long, Long> statistics,
- Map<Long, Double> size, Map<Long, Long> hitFrequencyMap, Map<Long, Map<Long, Long>> scanCountSourceMap) {
+ private CuboidStats(String key, long baseCuboidId, double queryUncertaintyRatio, double bpusMinBenefitRatio,
+ Set<Long> mandatoryCuboids, Map<Long, Long> statistics, Map<Long, Double> size,
+ Map<Long, Long> hitFrequencyMap, Map<Long, Map<Long, Long>> scanCountSourceMap) {
this.key = key;
this.baseCuboid = baseCuboidId;
+ this.bpusMinBenefitRatio = bpusMinBenefitRatio;
/** Initial mandatory cuboids */
Set<Long> cuboidsForMandatory = Sets.newHashSet(mandatoryCuboids);
//Always add base cuboid.
@@ -141,44 +175,19 @@ private CuboidStats(String key, long baseCuboidId, Set<Long> mandatoryCuboids, M
cuboidsForSelection.removeAll(cuboidsForMandatory);
//There's no overlap between mandatoryCuboidSet and selectionCuboidSet
- this.mandatoryCuboidSet = ImmutableSet.<Long>builder().addAll(cuboidsForMandatory).build();
- this.selectionCuboidSet = ImmutableSet.<Long>builder().addAll(cuboidsForSelection).build();
+ this.mandatoryCuboidSet = ImmutableSet.<Long> builder().addAll(cuboidsForMandatory).build();
+ this.selectionCuboidSet = ImmutableSet.<Long> builder().addAll(cuboidsForSelection).build();
if (selectionCuboidSet.isEmpty()) {
logger.warn("The selection set should not be empty!!!");
}
- /** Initialize row count for mandatory cuboids */
- CuboidStatsUtil.complementRowCountForMandatoryCuboids(statistics, baseCuboid, mandatoryCuboidSet);
-
- this.cuboidCountMap = ImmutableMap.<Long, Long>builder().putAll(statistics).build();
- this.cuboidSizeMap = ImmutableMap.<Long, Double>builder().putAll(size).build();
+ this.cuboidCountMap = ImmutableMap.<Long, Long> builder().putAll(statistics).build();
+ this.cuboidSizeMap = ImmutableMap.<Long, Double> builder().putAll(size).build();
/** Initialize the hit probability for each selection cuboid */
- Map<Long, Double> tmpCuboidHitProbabilityMap = Maps.newHashMapWithExpectedSize(selectionCuboidSet.size());
- if (hitFrequencyMap != null) {
- long totalHitFrequency = 0L;
- for (Map.Entry<Long, Long> hitFrequency : hitFrequencyMap.entrySet()) {
- if (selectionCuboidSet.contains(hitFrequency.getKey())) {
- totalHitFrequency += hitFrequency.getValue();
- }
- }
-
- final double unitUncertainProb = WEIGHT_FOR_UN_QUERY / selectionCuboidSet.size();
- for (Long cuboid : selectionCuboidSet) {
- //Calculate hit probability for each cuboid
- if (hitFrequencyMap.get(cuboid) != null) {
- tmpCuboidHitProbabilityMap.put(cuboid, unitUncertainProb
- + (1 - WEIGHT_FOR_UN_QUERY) * hitFrequencyMap.get(cuboid) / totalHitFrequency);
- } else {
- tmpCuboidHitProbabilityMap.put(cuboid, unitUncertainProb);
- }
- }
- } else {
- for (Long cuboid : selectionCuboidSet) {
- tmpCuboidHitProbabilityMap.put(cuboid, 1.0 / selectionCuboidSet.size());
- }
- }
- this.cuboidHitProbabilityMap = ImmutableMap.<Long, Double>builder().putAll(tmpCuboidHitProbabilityMap).build();
+ Map<Long, Double> tmpCuboidHitProbabilityMap = CuboidStatsUtil.calculateCuboidHitProbability(selectionCuboidSet,
+ hitFrequencyMap, selectionCuboidSet.size(), queryUncertaintyRatio);
+ this.cuboidHitProbabilityMap = ImmutableMap.<Long, Double> builder().putAll(tmpCuboidHitProbabilityMap).build();
/** Initialize the scan count when query for each selection cuboid + one base cuboid */
Map<Long, Long> tmpCuboidScanCountMap = Maps.newHashMapWithExpectedSize(1 + selectionCuboidSet.size());
@@ -186,16 +195,16 @@ private CuboidStats(String key, long baseCuboidId, Set<Long> mandatoryCuboids, M
for (Long cuboid : selectionCuboidSet) {
tmpCuboidScanCountMap.put(cuboid, getExpScanCount(cuboid, statistics, scanCountSourceMap));
}
- this.cuboidScanCountMap = ImmutableMap.<Long, Long>builder().putAll(tmpCuboidScanCountMap).build();
+ this.cuboidScanCountMap = ImmutableMap.<Long, Long> builder().putAll(tmpCuboidScanCountMap).build();
- this.directChildrenCache = ImmutableMap.<Long, List<Long>>builder()
+ this.directChildrenCache = ImmutableMap.<Long, List<Long>> builder()
.putAll(CuboidStatsUtil.createDirectChildrenCache(statistics.keySet())).build();
this.allDescendantsCache = Maps.newConcurrentMap();
}
private long getExpScanCount(long sourceCuboid, Map<Long, Long> statistics,
- Map<Long, Map<Long, Long>> scanCountSourceMap) {
+ Map<Long, Map<Long, Long>> scanCountSourceMap) {
Preconditions.checkNotNull(statistics.get(sourceCuboid),
"The statistics for source cuboid " + sourceCuboid + " does not exist!!!");
if (scanCountSourceMap == null || scanCountSourceMap.get(sourceCuboid) == null
@@ -216,6 +225,10 @@ private long getExpScanCount(long sourceCuboid, Map<Long, Long> statistics,
}
}
+ public double getBpusMinBenefitRatio() {
+ return bpusMinBenefitRatio;
+ }
+
public Set<Long> getAllDescendants(long cuboid) {
Set<Long> allDescendants = Sets.newLinkedHashSet();
if (selectionCuboidSet.contains(cuboid)) {
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
index dc3471b4b7..90eafdd275 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
@@ -27,81 +27,208 @@
import java.util.SortedSet;
import java.util.TreeSet;
+import org.apache.kylin.common.util.Pair;
+
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
public class CuboidStatsUtil {
/**
- * For generating mandatory cuboids,
- * a cuboid is mandatory if the expectation of rolling up count exceeds a threshold
+ * According to the cuboid hit frequencies and query uncertainty ratio
+ * calculate each cuboid hit probability
+ * @param selectionCuboidSet subset of cuboid domain which needs probability
+ * @param nTotalCuboids number of cuboids needs to be considered, mainly for each cuboid's uncertainty weight
* */
- public static Set<Long> generateMandatoryCuboidSet(Map<Long, Long> statistics, Map<Long, Long> hitFrequencyMap,
- Map<Long, Map<Long, Long>> rollingUpCountSourceMap, final long rollUpThresholdForMandatory) {
- Set<Long> mandatoryCuboidSet = Sets.newHashSet();
- if (hitFrequencyMap == null || hitFrequencyMap.isEmpty() || rollingUpCountSourceMap == null
- || rollingUpCountSourceMap.isEmpty()) {
- return mandatoryCuboidSet;
- }
- long totalHitFrequency = 0L;
- for (long hitFrequency : hitFrequencyMap.values()) {
- totalHitFrequency += hitFrequency;
+ public static Map<Long, Double> calculateCuboidHitProbability(Set<Long> selectionCuboidSet,
+ Map<Long, Long> hitFrequencyMap, long nTotalCuboids, double queryUncertaintyRatio) {
+ Map<Long, Double> cuboidHitProbabilityMap = Maps.newHashMapWithExpectedSize(selectionCuboidSet.size());
+ if (hitFrequencyMap == null || hitFrequencyMap.isEmpty()) {
+ for (Long cuboid : selectionCuboidSet) {
+ cuboidHitProbabilityMap.put(cuboid, 1.0 / nTotalCuboids);
+ }
+ } else {
+ long totalHitFrequency = 0L;
+ for (Map.Entry<Long, Long> hitFrequency : hitFrequencyMap.entrySet()) {
+ totalHitFrequency += hitFrequency.getValue();
+ }
+
+ final double unitUncertainProb = queryUncertaintyRatio / nTotalCuboids;
+ for (Long cuboid : selectionCuboidSet) {
+ //Calculate hit probability for each cuboid
+ if (hitFrequencyMap.get(cuboid) != null) {
+ cuboidHitProbabilityMap.put(cuboid, unitUncertainProb
+ + (1 - queryUncertaintyRatio) * hitFrequencyMap.get(cuboid) / totalHitFrequency);
+ } else {
+ cuboidHitProbabilityMap.put(cuboid, unitUncertainProb);
+ }
+ }
}
- if (totalHitFrequency == 0) {
- return mandatoryCuboidSet;
+ return cuboidHitProbabilityMap;
+ }
+
+ /**
+ * @param statistics for cuboid row count
+ * @param rollingUpSourceMap the key of the outer map is source cuboid,
+ * the key of the inner map is target cuboid,
+ * if cube is optimized multiple times, target cuboid may change
+ * the first element of the pair is the rollup row count
+ * the second element of the pair is the return row count
+ * @return source cuboids with estimated row count
+ */
+ public static Map<Long, Long> generateSourceCuboidStats(Map<Long, Long> statistics,
+ Map<Long, Double> cuboidHitProbabilityMap, Map<Long, Map<Long, Pair<Long, Long>>> rollingUpSourceMap) {
+ Map<Long, Long> srcCuboidsStats = Maps.newHashMap();
+ if (cuboidHitProbabilityMap == null || cuboidHitProbabilityMap.isEmpty() || rollingUpSourceMap == null
+ || rollingUpSourceMap.isEmpty()) {
+ return srcCuboidsStats;
}
- for (Map.Entry<Long, Long> hitFrequency : hitFrequencyMap.entrySet()) {
- long cuboid = hitFrequency.getKey();
+ for (Long cuboid : cuboidHitProbabilityMap.keySet()) {
+ if (statistics.get(cuboid) != null) {
+ continue;
+ }
+ Map<Long, Pair<Long, Long>> innerRollingUpTargetMap = rollingUpSourceMap.get(cuboid);
+ if (innerRollingUpTargetMap == null || innerRollingUpTargetMap.isEmpty()) {
+ continue;
+ }
- if (isCuboidMandatory(cuboid, statistics, rollingUpCountSourceMap)) {
- long totalEstScanCount = 0L;
- for (long estScanCount : rollingUpCountSourceMap.get(cuboid).values()) {
- totalEstScanCount += estScanCount;
- }
- totalEstScanCount /= rollingUpCountSourceMap.get(cuboid).size();
- if ((hitFrequency.getValue() * 1.0 / totalHitFrequency)
- * totalEstScanCount >= rollUpThresholdForMandatory) {
- mandatoryCuboidSet.add(cuboid);
+ long totalEstRowCount = 0L;
+ int nEffective = 0;
+ boolean ifHasStats = false;
+ // if ifHasStats equals true, then source cuboid row count = (1 - rollup ratio) * target cuboid row count
+ // else source cuboid row count = returned row count collected directly
+ for (Long tgtCuboid : innerRollingUpTargetMap.keySet()) {
+ Pair<Long, Long> rollingupStats = innerRollingUpTargetMap.get(tgtCuboid);
+ if (statistics.get(tgtCuboid) != null) {
+ if (!ifHasStats) {
+ totalEstRowCount = 0L;
+ nEffective = 0;
+ ifHasStats = true;
+ }
+ double rollupRatio = calculateRollupRatio(rollingupStats);
+ totalEstRowCount += (1 - rollupRatio) * statistics.get(tgtCuboid);
+ nEffective++;
+ } else {
+ if (ifHasStats) {
+ continue;
+ }
+ totalEstRowCount += rollingupStats.getSecond();
+ nEffective++;
}
}
- }
- return mandatoryCuboidSet;
- }
- private static boolean isCuboidMandatory(Long cuboid, Map<Long, Long> statistics, Map<Long, Map<Long, Long>> rollingUpCountSourceMap) {
- return !statistics.containsKey(cuboid) && rollingUpCountSourceMap.containsKey(cuboid) && !rollingUpCountSourceMap.get(cuboid).isEmpty();
+ srcCuboidsStats.put(cuboid, totalEstRowCount / nEffective);
+ }
+ srcCuboidsStats.remove(0L);
+ adjustCuboidStats(srcCuboidsStats, statistics);
+ return srcCuboidsStats;
}
/**
* Complement row count for mandatory cuboids
* with its best parent's row count
* */
- public static void complementRowCountForMandatoryCuboids(Map<Long, Long> statistics, long baseCuboid,
- Set<Long> mandatoryCuboidSet) {
+ public static Map<Long, Long> complementRowCountForCuboids(final Map<Long, Long> statistics, Set<Long> cuboids) {
+ Map<Long, Long> result = Maps.newHashMapWithExpectedSize(cuboids.size());
+
// Sort entries order by row count asc
- SortedSet<Map.Entry<Long, Long>> sortedStatsSet = new TreeSet<>(
- new Comparator<Map.Entry<Long, Long>>() {
- public int compare(Map.Entry<Long, Long> o1, Map.Entry<Long, Long> o2) {
- return o1.getValue().compareTo(o2.getValue());
- }
- });
- //sortedStatsSet.addAll(statistics.entrySet()); KYLIN-3580
- for(Map.Entry<Long, Long> entry : statistics.entrySet()){
- sortedStatsSet.add(entry);
- }
- for (Long cuboid : mandatoryCuboidSet) {
+ SortedSet<Map.Entry<Long, Long>> sortedStatsSet = new TreeSet<>(new Comparator<Map.Entry<Long, Long>>() {
+ public int compare(Map.Entry<Long, Long> o1, Map.Entry<Long, Long> o2) {
+ int ret = o1.getValue().compareTo(o2.getValue());
+ return ret == 0 ? o1.getKey().compareTo(o2.getKey()) : ret;
+ }
+ });
+ sortedStatsSet.addAll(statistics.entrySet());
+ for (Long cuboid : cuboids) {
if (statistics.get(cuboid) == null) {
// Get estimate row count for mandatory cuboid
- long tmpRowCount = -1;
for (Map.Entry<Long, Long> entry : sortedStatsSet) {
if (isDescendant(cuboid, entry.getKey())) {
- tmpRowCount = entry.getValue();
+ result.put(cuboid, entry.getValue());
+ break;
+ }
+ }
+ } else {
+ result.put(cuboid, statistics.get(cuboid));
+ }
+ }
+
+ return result;
+ }
+
+ /**
+ * adjust cuboid row count, make sure parent not less than child
+ */
+ public static Map<Long, Long> adjustCuboidStats(Map<Long, Long> statistics) {
+ Map<Long, Long> ret = Maps.newHashMapWithExpectedSize(statistics.size());
+
+ List<Long> cuboids = Lists.newArrayList(statistics.keySet());
+ Collections.sort(cuboids);
+
+ for (Long cuboid : cuboids) {
+ Long rowCount = statistics.get(cuboid);
+ for (Long childCuboid : ret.keySet()) {
+ if (isDescendant(childCuboid, cuboid)) {
+ Long childRowCount = ret.get(childCuboid);
+ if (rowCount < childRowCount) {
+ rowCount = childRowCount;
+ }
+ }
+ }
+ ret.put(cuboid, rowCount);
+ }
+
+ return ret;
+ }
+
+ public static void adjustCuboidStats(Map<Long, Long> mandatoryCuboidsWithStats, Map<Long, Long> statistics) {
+ List<Long> mandatoryCuboids = Lists.newArrayList(mandatoryCuboidsWithStats.keySet());
+ Collections.sort(mandatoryCuboids);
+
+ List<Long> selectedCuboids = Lists.newArrayList(statistics.keySet());
+ Collections.sort(selectedCuboids);
+
+ for (int i = 0; i < mandatoryCuboids.size(); i++) {
+ Long mCuboid = mandatoryCuboids.get(i);
+ if (statistics.get(mCuboid) != null) {
+ mandatoryCuboidsWithStats.put(mCuboid, statistics.get(mCuboid));
+ continue;
+ }
+ int k = 0;
+ // Make sure mCuboid's row count larger than its children's row count in statistics
+ for (; k < selectedCuboids.size(); k++) {
+ Long sCuboid = selectedCuboids.get(k);
+ if (sCuboid > mCuboid) {
+ break;
+ }
+ if (isDescendant(sCuboid, mCuboid)) {
+ Long childRowCount = statistics.get(sCuboid);
+ if (childRowCount > mandatoryCuboidsWithStats.get(mCuboid)) {
+ mandatoryCuboidsWithStats.put(mCuboid, childRowCount);
+ }
+ }
+ }
+ // Make sure mCuboid's row count larger than its children's row count in mandatoryCuboids
+ for (int j = 0; j < i; j++) {
+ Long cCuboid = mandatoryCuboids.get(j);
+ if (isDescendant(cCuboid, mCuboid)) {
+ Long childRowCount = mandatoryCuboidsWithStats.get(cCuboid);
+ if (childRowCount > mandatoryCuboidsWithStats.get(mCuboid)) {
+ mandatoryCuboidsWithStats.put(mCuboid, childRowCount);
+ }
+ }
+ }
+ // Make sure mCuboid's row count lower than its parents' row count in statistics
+ for (; k < selectedCuboids.size(); k++) {
+ Long sCuboid = selectedCuboids.get(k);
+ if (isDescendant(mCuboid, sCuboid)) {
+ Long parentRowCount = statistics.get(sCuboid);
+ if (parentRowCount < mandatoryCuboidsWithStats.get(mCuboid)) {
+ mandatoryCuboidsWithStats.put(mCuboid, parentRowCount);
}
}
- statistics.put(cuboid, tmpRowCount < 0 ? statistics.get(baseCuboid) : tmpRowCount);
}
}
}
@@ -207,4 +334,9 @@ private static void checkAndAddDirectChild(List<Long> directChildren, Long curre
public static boolean isDescendant(long cuboidToCheck, long parentCuboid) {
return (cuboidToCheck & parentCuboid) == cuboidToCheck;
}
+
+ private static double calculateRollupRatio(Pair<Long, Long> rollupStats) {
+ double rollupInputCount = rollupStats.getFirst() + rollupStats.getSecond();
+ return rollupInputCount == 0 ? 0 : 1.0 * rollupStats.getFirst() / rollupInputCount;
+ }
}
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
index ba8833afe8..1ab96a25d9 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
@@ -23,6 +23,7 @@
import java.util.Map;
import java.util.Set;
+import org.apache.kylin.common.util.Pair;
import org.junit.Assert;
import org.junit.Test;
@@ -44,6 +45,9 @@
* / | /
* 00000100 00000010
* */
+
+ private final static long baseCuboidId = 255L;
+
private Set<Long> generateCuboidSet() {
return Sets.newHashSet(255L, 159L, 239L, 50L, 199L, 6L, 4L, 2L);
}
@@ -53,11 +57,11 @@
countMap.put(255L, 10000L);
countMap.put(159L, 10000L);
- countMap.put(50L, 10000L);
- countMap.put(199L, 10000L);
- countMap.put(6L, 10000L);
- countMap.put(4L, 10000L);
- countMap.put(2L, 10000L);
+ countMap.put(50L, 800L);
+ countMap.put(199L, 200L);
+ countMap.put(6L, 60L);
+ countMap.put(4L, 40L);
+ countMap.put(2L, 20L);
return countMap;
}
@@ -67,26 +71,46 @@
long totalHitFrequency = 10000L;
- hitFrequencyMap.put(239L, (long) (totalHitFrequency * 0.5));
+ hitFrequencyMap.put(239L, (long) (totalHitFrequency * 0.2));
hitFrequencyMap.put(50L, (long) (totalHitFrequency * 0.2));
- hitFrequencyMap.put(2L, (long) (totalHitFrequency * 0.25));
+ hitFrequencyMap.put(2L, (long) (totalHitFrequency * 0.24));
hitFrequencyMap.put(178L, (long) (totalHitFrequency * 0.05));
+ hitFrequencyMap.put(187L, (long) (totalHitFrequency * 0.3));
+ hitFrequencyMap.put(0L, (long) (totalHitFrequency * 0.01));
return hitFrequencyMap;
}
- private Map<Long, Map<Long, Long>> simulateRollingUpCount() {
- Map<Long, Map<Long, Long>> rollingUpCountMap = Maps.newLinkedHashMap();
+ private Map<Long, Double> simulateHitProbability(long nCuboids) {
+ Map<Long, Long> hitFrequencyMap = simulateHitFrequency();
+ return CuboidStatsUtil.calculateCuboidHitProbability(hitFrequencyMap.keySet(), hitFrequencyMap, nCuboids,
+ CuboidStats.WEIGHT_FOR_UN_QUERY);
+ }
+
+ private Map<Long, Map<Long, Pair<Long, Long>>> simulateRollingUpCount() {
+ Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountMap = Maps.newLinkedHashMap();
+
+ rollingUpCountMap.put(239L, new HashMap<Long, Pair<Long, Long>>() {
+ {
+ put(255L, new Pair<>(990L, 10L));
+ }
+ });
+
+ rollingUpCountMap.put(178L, new HashMap<Long, Pair<Long, Long>>() {
+ {
+ put(255L, new Pair<>(4999L, 1L));
+ }
+ });
- rollingUpCountMap.put(239L, new HashMap<Long, Long>() {
+ rollingUpCountMap.put(187L, new HashMap<Long, Pair<Long, Long>>() {
{
- put(255L, 4000L);
+ put(251L, new Pair<>(3000L, 1000L));
}
});
- rollingUpCountMap.put(178L, new HashMap<Long, Long>() {
+ rollingUpCountMap.put(0L, new HashMap<Long, Pair<Long, Long>>() {
{
- put(255L, 5000L);
+ put(2L, new Pair<>(19L, 1L));
}
});
@@ -101,25 +125,113 @@ public void isDescendantTest() {
@Test
public void generateMandatoryCuboidSetTest() {
- Set<Long> mandatoryCuboidSet = CuboidStatsUtil.generateMandatoryCuboidSet(simulateCount(),
- simulateHitFrequency(), simulateRollingUpCount(), 1000L);
- Assert.assertTrue(mandatoryCuboidSet.contains(239L));
- Assert.assertTrue(!mandatoryCuboidSet.contains(178L));
+ Map<Long, Long> srcCuboidSet = CuboidStatsUtil.generateSourceCuboidStats(simulateCount(),
+ simulateHitProbability(baseCuboidId), simulateRollingUpCount());
+
+ Assert.assertTrue(srcCuboidSet.get(239L) == 200L);
+ Assert.assertTrue(srcCuboidSet.get(187L) == 1000L);
+ Assert.assertTrue(srcCuboidSet.get(178L) == 800L);
+
+ Assert.assertTrue(!srcCuboidSet.containsKey(0L));
}
@Test
public void complementRowCountForMandatoryCuboidsTest() {
Map<Long, Long> countMap = simulateCount();
- Set<Long> mandatoryCuboidSet = CuboidStatsUtil.generateMandatoryCuboidSet(countMap, simulateHitFrequency(),
- simulateRollingUpCount(), 1000L);
- for (long mandatoryCuboid : mandatoryCuboidSet) {
+ Map<Long, Long> srcCuboidsStats = CuboidStatsUtil.generateSourceCuboidStats(countMap,
+ simulateHitProbability(baseCuboidId), simulateRollingUpCount());
+ for (long mandatoryCuboid : srcCuboidsStats.keySet()) {
Assert.assertNull(countMap.get(mandatoryCuboid));
}
- CuboidStatsUtil.complementRowCountForMandatoryCuboids(countMap, 255L, mandatoryCuboidSet);
- for (long mandatoryCuboid : mandatoryCuboidSet) {
- Assert.assertNotNull(countMap.get(mandatoryCuboid));
- }
- Assert.assertTrue(countMap.get(239L) == 10000L);
+ Assert.assertTrue(srcCuboidsStats.get(239L) == 200L);
+
+ Map<Long, Long> mandatoryCuboidsWithStats2 = Maps.newHashMap();
+ mandatoryCuboidsWithStats2.put(215L, countMap.get(255L));
+ mandatoryCuboidsWithStats2.put(34L, countMap.get(50L));
+ Assert.assertEquals(mandatoryCuboidsWithStats2,
+ CuboidStatsUtil.complementRowCountForCuboids(countMap, mandatoryCuboidsWithStats2.keySet()));
+ }
+
+ @Test
+ public void testAdjustMandatoryCuboidStats() {
+ Map<Long, Long> statistics = Maps.newHashMap();
+ statistics.put(60160L, 1212L);
+
+ Map<Long, Long> cuboidsWithStats = Maps.newHashMap();
+ cuboidsWithStats.put(65280L, 1423L);
+ cuboidsWithStats.put(63232L, 2584421L);
+ cuboidsWithStats.put(61184L, 132L);
+ cuboidsWithStats.put(57088L, 499L);
+ cuboidsWithStats.put(55040L, 708L);
+ cuboidsWithStats.put(38656L, 36507L);
+
+ Map<Long, Double> cuboidHitProbabilityMap = Maps.newHashMap();
+ cuboidHitProbabilityMap.put(65280L, 0.2);
+ cuboidHitProbabilityMap.put(63232L, 0.16);
+ cuboidHitProbabilityMap.put(61184L, 0.16);
+ cuboidHitProbabilityMap.put(57088L, 0.16);
+ cuboidHitProbabilityMap.put(55040L, 0.16);
+ cuboidHitProbabilityMap.put(38656L, 0.16);
+
+ Map<Long, Long> cuboidsWithStatsExpected = Maps.newHashMap(cuboidsWithStats);
+ cuboidsWithStatsExpected.put(65280L, 2584421L);
+ cuboidsWithStatsExpected.put(57088L, 36507L);
+ cuboidsWithStatsExpected.put(55040L, 36507L);
+ cuboidsWithStatsExpected.put(61184L, 1212L);
+
+ CuboidStatsUtil.adjustCuboidStats(cuboidsWithStats, statistics);
+ Assert.assertEquals(cuboidsWithStatsExpected, cuboidsWithStats);
+ }
+
+ /**
+ * 1111(70) 1111(90)
+ * / \ / \
+ * 1011(90) \ 1111(90) \
+ * | \ | \
+ * 0011(40) 1110(50) ==========> 0011(80) 1110(80)
+ * / \ / \ / \ / \
+ * 0001(20) 0010(80) 0100(60) 0001(20) 0010(80) 0100(60)
+ *
+ *
+ * +
+ *
+ *
+ * / \
+ * / \
+ * / 1001(85)
+ * / \
+ * 0111(70) 1000(100)
+ * */
+ @Test
+ public void testAdjustCuboidStats() {
+ Map<Long, Long> statistics = Maps.newHashMap();
+ statistics.put(1L, 20L);
+ statistics.put(2L, 80L);
+ statistics.put(4L, 60L);
+ statistics.put(3L, 40L);
+ statistics.put(11L, 90L);
+ statistics.put(14L, 50L);
+ statistics.put(15L, 70L);
+
+ Map<Long, Long> cuboidsWithStatsExpected = Maps.newHashMap(statistics);
+ cuboidsWithStatsExpected.put(3L, 80L);
+ cuboidsWithStatsExpected.put(14L, 80L);
+ cuboidsWithStatsExpected.put(15L, 90L);
+
+ statistics = CuboidStatsUtil.adjustCuboidStats(statistics);
+ Assert.assertEquals(cuboidsWithStatsExpected, statistics);
+
+ Map<Long, Long> mandatoryCuboidsWithStats = Maps.newHashMap();
+ mandatoryCuboidsWithStats.put(7L, 70L);
+ mandatoryCuboidsWithStats.put(8L, 100L);
+ mandatoryCuboidsWithStats.put(9L, 85L);
+ CuboidStatsUtil.adjustCuboidStats(mandatoryCuboidsWithStats, statistics);
+
+ Map<Long, Long> mandatoryCuboidsWithStatsExpected = Maps.newHashMap();
+ mandatoryCuboidsWithStatsExpected.put(7L, 80L);
+ mandatoryCuboidsWithStatsExpected.put(8L, 80L);
+ mandatoryCuboidsWithStatsExpected.put(9L, 85L);
+ Assert.assertEquals(mandatoryCuboidsWithStatsExpected, mandatoryCuboidsWithStats);
}
@Test
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
index 05458b64fb..2bffe86560 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
@@ -22,9 +22,11 @@
import java.util.Map;
import java.util.Set;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.cuboid.algorithm.CuboidRecommender;
import org.apache.kylin.cube.cuboid.algorithm.CuboidStats;
import org.slf4j.Logger;
@@ -65,23 +67,45 @@
/** Trigger cube planner phase two for optimization */
public static Map<Long, Long> getRecommendCuboidList(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
- Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
+ Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap) throws IOException {
+ CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+ Set<Long> currentCuboids = cuboidScheduler.getAllCuboidIds();
Pair<Map<Long, Long>, Map<Long, Double>> statsPair = CuboidStatsReaderUtil
- .readCuboidStatsAndSizeFromCube(cube.getCuboidScheduler().getAllCuboidIds(), cube);
+ .readCuboidStatsAndSizeFromCube(currentCuboids, cube);
+ long baseCuboid = cuboidScheduler.getBaseCuboidId();
+ if (statsPair.getFirst().get(baseCuboid) == null || statsPair.getFirst().get(baseCuboid) == 0L) {
+ logger.info("Base cuboid count in cuboid statistics is 0.");
+ return null;
+ }
+ KylinConfig config = cube.getConfig();
String key = cube.getName();
- long baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
- CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, statsPair.getFirst(), statsPair.getSecond())
- .setHitFrequencyMap(hitFrequencyMap).setRollingUpCountSourceMap(rollingUpCountSourceMap,
- cube.getConfig().getCubePlannerMandatoryRollUpThreshold())
+ double queryUncertaintyRatio = config.getCubePlannerQueryUncertaintyRatio();
+ double bpusMinBenefitRatio = config.getCubePlannerBPUSMinBenefitRatio();
+ CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, statsPair.getFirst(),
+ statsPair.getSecond()) {
+ @Override
+ public Map<Long, Double> estimateCuboidsSize(Map<Long, Long> statistics) {
+ try {
+ return CuboidStatsReaderUtil.readCuboidSizeFromCube(statistics, cube);
+ } catch (IOException e) {
+ logger.warn("Fail to get cuboid size from cube due to ", e);
+ return null;
+ }
+ }
+ }.setQueryUncertaintyRatio(queryUncertaintyRatio) //
+ .setBPUSMinBenefitRatio(bpusMinBenefitRatio) //
+ .setHitFrequencyMap(hitFrequencyMap) //
+ .setRollingUpCountSourceMap(rollingUpCountSourceMap) //
.build();
- return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, cube.getConfig());
+ return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, config);
}
/** For future segment level recommend */
public static Map<Long, Long> getRecommendCuboidList(CubeSegment segment, Map<Long, Long> hitFrequencyMap,
- Map<Long, Map<Long, Long>> rollingUpCountSourceMap, boolean ifForceRecommend) throws IOException {
+ Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap, boolean ifForceRecommend)
+ throws IOException {
if (segment == null) {
return null;
}
@@ -103,9 +127,7 @@
String key = cube.getName() + "-" + segment.getName();
CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, cubeStatsReader.getCuboidRowEstimatesHLL(),
cubeStatsReader.getCuboidSizeMap()).setHitFrequencyMap(hitFrequencyMap)
- .setRollingUpCountSourceMap(rollingUpCountSourceMap,
- segment.getConfig().getCubePlannerMandatoryRollUpThreshold())
- .build();
+ .setRollingUpCountSourceMap(rollingUpCountSourceMap).build();
return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, segment.getConfig(),
ifForceRecommend);
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
index 1542aa2e6d..ee615c3dbe 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
@@ -68,6 +68,29 @@
return statisticsMerged.isEmpty() ? null : statisticsMerged;
}
+ public static Map<Long, Double> readCuboidSizeFromCube(Map<Long, Long> statistics, CubeInstance cube)
+ throws IOException {
+ List<CubeSegment> segmentList = cube.getSegments(SegmentStatusEnum.READY);
+ Map<Long, Double> sizeMerged = Maps.newHashMapWithExpectedSize(statistics.size());
+ for (CubeSegment pSegment : segmentList) {
+ CubeStatsReader pReader = new CubeStatsReader(pSegment, null, pSegment.getConfig());
+ Map<Long, Double> pSizeMap = CubeStatsReader.getCuboidSizeMapFromRowCount(pSegment, statistics,
+ pReader.sourceRowCount);
+ for (Long pCuboid : statistics.keySet()) {
+ Double pSize = sizeMerged.get(pCuboid);
+ sizeMerged.put(pCuboid, pSize == null ? pSizeMap.get(pCuboid) : pSize + pSizeMap.get(pCuboid));
+ }
+ }
+ int nSegment = segmentList.size();
+ if (nSegment <= 1) {
+ return sizeMerged;
+ }
+ for (Long pCuboid : statistics.keySet()) {
+ sizeMerged.put(pCuboid, sizeMerged.get(pCuboid) / nSegment);
+ }
+ return sizeMerged;
+ }
+
private static void readCuboidStatsFromSegments(Set<Long> cuboidSet, List<CubeSegment> segmentList,
final Map<Long, Long> statisticsMerged, final Map<Long, Double> sizeMerged) throws IOException {
if (segmentList == null || segmentList.isEmpty()) {
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index dcaa259e19..08b03a48f3 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -32,6 +32,7 @@
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -56,8 +57,6 @@
import org.apache.kylin.metadata.model.SegmentRange.TSRange;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metrics.MetricsManager;
-import org.apache.kylin.metrics.property.QueryCubePropertyEnum;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -70,7 +69,6 @@
import org.apache.kylin.rest.request.JobBuildRequest2;
import org.apache.kylin.rest.request.JobOptimizeRequest;
import org.apache.kylin.rest.request.LookupSnapshotBuildRequest;
-import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.EnvelopeResponse;
@@ -80,7 +78,6 @@
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
import org.apache.kylin.rest.service.ProjectService;
-import org.apache.kylin.rest.service.QueryService;
import org.apache.kylin.rest.util.ValidateUtil;
import org.apache.kylin.source.kafka.util.KafkaClient;
import org.slf4j.Logger;
@@ -124,10 +121,6 @@
@Qualifier("projectService")
private ProjectService projectService;
- @Autowired
- @Qualifier("queryService")
- private QueryService queryService;
-
@RequestMapping(value = "/validate/{cubeName}", method = RequestMethod.GET, produces = { "application/json" })
@ResponseBody
public EnvelopeResponse<Boolean> validateModelName(@PathVariable String cubeName) {
@@ -873,7 +866,7 @@ public CuboidTreeResponse getCurrentCuboids(@PathVariable String cubeName) {
Map<Long, Long> queryMatchMap = null;
try {
hitFrequencyMap = getTargetCuboidHitFrequency(cubeName);
- queryMatchMap = getCuboidQueryMatchCount(cubeName);
+ queryMatchMap = cubeService.getCuboidQueryMatchCount(cubeName);
} catch (Exception e) {
logger.warn("Fail to query on system cube due to " + e);
}
@@ -899,7 +892,7 @@ public CuboidTreeResponse getRecommendCuboids(@PathVariable String cubeName) thr
// Get cuboid target info for displaying heat map of cuboid hit
Map<Long, Long> displayHitFrequencyMap = getTargetCuboidHitFrequency(cubeName);
// Get exactly matched cuboid query count
- Map<Long, Long> queryMatchMap = getCuboidQueryMatchCount(cubeName);
+ Map<Long, Long> queryMatchMap = cubeService.getCuboidQueryMatchCount(cubeName);
Set<Long> currentCuboidSet = cube.getCuboidScheduler().getAllCuboidIds();
return cubeService.getCuboidTreeResponse(cuboidScheduler, recommendCuboidStatsMap, displayHitFrequencyMap,
@@ -909,69 +902,17 @@ public CuboidTreeResponse getRecommendCuboids(@PathVariable String cubeName) thr
private Map<Long, Long> getRecommendCuboidList(CubeInstance cube) throws IOException {
// Get cuboid source info
Map<Long, Long> optimizeHitFrequencyMap = getSourceCuboidHitFrequency(cube.getName());
- Map<Long, Map<Long, Long>> rollingUpCountSourceMap = getCuboidRollingUpCount(cube.getName());
+ Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap = cubeService
+ .getCuboidRollingUpStats(cube.getName());
return cubeService.getRecommendCuboidStatistics(cube, optimizeHitFrequencyMap, rollingUpCountSourceMap);
}
private Map<Long, Long> getSourceCuboidHitFrequency(String cubeName) {
- return getCuboidHitFrequency(cubeName, true);
+ return cubeService.getCuboidHitFrequency(cubeName, true);
}
private Map<Long, Long> getTargetCuboidHitFrequency(String cubeName) {
- return getCuboidHitFrequency(cubeName, false);
- }
-
- private Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean isCuboidSource) {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
- String cuboidColumn = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
- if (!isCuboidSource) {
- cuboidColumn = QueryCubePropertyEnum.CUBOID_TARGET.toString();
- }
- String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
- String table = cubeService.getMetricsManager()
- .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
- String sql = "select " + cuboidColumn + ", sum(" + hitMeasure + ") " //
- + "from " + table//
- + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " //
- + "group by " + cuboidColumn;
- sqlRequest.setSql(sql);
- List<List<String>> orgHitFrequency = queryService.doQueryWithCache(sqlRequest).getResults();
- return cubeService.formatQueryCount(orgHitFrequency);
- }
-
- private Map<Long, Map<Long, Long>> getCuboidRollingUpCount(String cubeName) {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
- String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
- String cuboidTarget = QueryCubePropertyEnum.CUBOID_TARGET.toString();
- String aggCount = QueryCubePropertyEnum.AGGR_COUNT.toString();
- String table = cubeService.getMetricsManager()
- .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
- String sql = "select " + cuboidSource + ", " + cuboidTarget + ", sum(" + aggCount + ")/count(*) " //
- + "from " + table //
- + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " //
- + "group by " + cuboidSource + ", " + cuboidTarget;
- sqlRequest.setSql(sql);
- List<List<String>> orgRollingUpCount = queryService.doQueryWithCache(sqlRequest).getResults();
- return cubeService.formatRollingUpCount(orgRollingUpCount);
- }
-
- private Map<Long, Long> getCuboidQueryMatchCount(String cubeName) {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
- String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
- String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
- String table = cubeService.getMetricsManager()
- .getSystemTableFromSubject(cubeService.getConfig().getKylinMetricsSubjectQueryCube());
- String sql = "select " + cuboidSource + ", sum(" + hitMeasure + ") " //
- + "from " + table //
- + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' and "
- + QueryCubePropertyEnum.IF_MATCH.toString() + " = true " //
- + "group by " + cuboidSource;
- sqlRequest.setSql(sql);
- List<List<String>> orgMatchHitFrequency = queryService.doQueryWithCache(sqlRequest).getResults();
- return cubeService.formatQueryCount(orgMatchHitFrequency);
+ return cubeService.getCuboidHitFrequency(cubeName, false);
}
/**
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
index ee9fdcdd01..846d6d311a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
@@ -16,15 +16,12 @@
* limitations under the License.
*/
-
package org.apache.kylin.rest.controller;
import java.util.List;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metrics.MetricsManager;
-import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.service.CubeService;
@@ -57,62 +54,69 @@
@RequestMapping(value = "/metric/cube", method = { RequestMethod.GET })
@ResponseBody
- public MetricsResponse getCubeMetrics(@RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "cubeName", required = false) String cubeName) {
+ public MetricsResponse getCubeMetrics(@RequestParam(value = "projectName", required = false) String projectName,
+ @RequestParam(value = "cubeName", required = false) String cubeName) {
checkAuthorization(projectName);
return dashboardService.getCubeMetrics(projectName, cubeName);
}
@RequestMapping(value = "/metric/query", method = RequestMethod.GET)
@ResponseBody
- public MetricsResponse getQueryMetrics(@RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
+ public MetricsResponse getQueryMetrics(@RequestParam(value = "projectName", required = false) String projectName,
+ @RequestParam(value = "cubeName", required = false) String cubeName,
+ @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
checkAuthorization(projectName);
MetricsResponse queryMetrics = new MetricsResponse();
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
String sql = dashboardService.getQueryMetricsSQL(startTime, endTime, projectName, cubeName);
- sqlRequest.setSql(sql);
- SQLResponse sqlResponse = queryService.doQueryWithCache(sqlRequest);
- if(!sqlResponse.getIsException()){
- queryMetrics.increase("queryCount", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
- queryMetrics.increase("avgQueryLatency", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
- queryMetrics.increase("maxQueryLatency", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
- queryMetrics.increase("minQueryLatency", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
+ SQLResponse sqlResponse = queryService.querySystemCube(sql);
+ if (!sqlResponse.getIsException()) {
+ queryMetrics.increase("queryCount",
+ dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
+ queryMetrics.increase("avgQueryLatency",
+ dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
+ queryMetrics.increase("maxQueryLatency",
+ dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
+ queryMetrics.increase("minQueryLatency",
+ dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
}
return queryMetrics;
}
@RequestMapping(value = "/metric/job", method = RequestMethod.GET)
@ResponseBody
- public MetricsResponse getJobMetrics(@RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
+ public MetricsResponse getJobMetrics(@RequestParam(value = "projectName", required = false) String projectName,
+ @RequestParam(value = "cubeName", required = false) String cubeName,
+ @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
checkAuthorization(projectName);
MetricsResponse jobMetrics = new MetricsResponse();
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
String sql = dashboardService.getJobMetricsSQL(startTime, endTime, projectName, cubeName);
- sqlRequest.setSql(sql);
- SQLResponse sqlResponse = queryService.doQueryWithCache(sqlRequest);
- if(!sqlResponse.getIsException()){
+ SQLResponse sqlResponse = queryService.querySystemCube(sql);
+ if (!sqlResponse.getIsException()) {
jobMetrics.increase("jobCount", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
- jobMetrics.increase("avgJobBuildTime", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
- jobMetrics.increase("maxJobBuildTime", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
- jobMetrics.increase("minJobBuildTime", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
+ jobMetrics.increase("avgJobBuildTime",
+ dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
+ jobMetrics.increase("maxJobBuildTime",
+ dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
+ jobMetrics.increase("minJobBuildTime",
+ dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
}
return jobMetrics;
}
@RequestMapping(value = "/chart/{category}/{metric}/{dimension}", method = RequestMethod.GET)
@ResponseBody
- public MetricsResponse getChartData(@PathVariable String dimension, @PathVariable String metric, @PathVariable String category, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
+ public MetricsResponse getChartData(@PathVariable String dimension, @PathVariable String metric,
+ @PathVariable String category, @RequestParam(value = "projectName", required = false) String projectName,
+ @RequestParam(value = "cubeName", required = false) String cubeName,
+ @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
checkAuthorization(projectName);
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
- String sql = dashboardService.getChartSQL(startTime, endTime, projectName, cubeName, dimension, metric, category);
- sqlRequest.setSql(sql);
- return dashboardService.transformChartData(queryService.doQueryWithCache(sqlRequest));
+ String sql = dashboardService.getChartSQL(startTime, endTime, projectName, cubeName, dimension, metric,
+ category);
+ return dashboardService.transformChartData(queryService.querySystemCube(sql));
}
- private void checkAuthorization(String projectName){
- if (projectName!=null && !projectName.isEmpty()) {
+ private void checkAuthorization(String projectName) {
+ if (projectName != null && !projectName.isEmpty()) {
ProjectInstance project = dashboardService.getProjectManager().getProject(projectName);
try {
dashboardService.checkAuthorization(project);
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index e541c37f14..a9fbb97787 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -32,6 +32,7 @@
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -65,6 +66,7 @@
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.metrics.property.QueryCubePropertyEnum;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
@@ -120,6 +122,10 @@
@Qualifier("modelMgmtService")
private ModelService modelService;
+ @Autowired
+ @Qualifier("queryService")
+ private QueryService queryService;
+
@Autowired
private AclEvaluate aclEvaluate;
@@ -472,7 +478,8 @@ public HBaseResponse getHTableInfo(String cubeName, String tableName) throws IOE
hr = new HBaseResponse();
CubeInstance cube = CubeManager.getInstance(getConfig()).getCube(cubeName);
- if (cube.getStorageType() == IStorageAware.ID_HBASE || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE) {
+ if (cube.getStorageType() == IStorageAware.ID_HBASE
+ || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE) {
try {
logger.debug("Loading HTable info " + cubeName + ", " + tableName);
@@ -547,9 +554,9 @@ public CubeInstance deleteSegment(CubeInstance cube, String segmentName) throws
}
public boolean isOrphonSegment(CubeInstance cube, String segId) {
- List<JobInstance> jobInstances = jobService.searchJobsByCubeName(cube.getName(),
- cube.getProject(), Lists.newArrayList(JobStatusEnum.NEW, JobStatusEnum.PENDING, JobStatusEnum.RUNNING,
- JobStatusEnum.ERROR, JobStatusEnum.STOPPED),
+ List<JobInstance> jobInstances = jobService.searchJobsByCubeName(
+ cube.getName(), cube.getProject(), Lists.newArrayList(JobStatusEnum.NEW, JobStatusEnum.PENDING,
+ JobStatusEnum.RUNNING, JobStatusEnum.ERROR, JobStatusEnum.STOPPED),
JobTimeFilterEnum.ALL, JobService.JobSearchMode.CUBING_ONLY);
for (JobInstance jobInstance : jobInstances) {
// if there are segment related jobs, can not delete this segment.
@@ -886,6 +893,12 @@ private NodeInfo generateNodeInfo(long cuboidId, int dimensionCount, long cubeQu
}
/** cube planner services */
+ public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
+ Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap) throws IOException {
+ aclEvaluate.checkProjectAdminPermission(cube.getProject());
+ return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap);
+ }
+
public Map<Long, Long> formatQueryCount(List<List<String>> orgQueryCount) {
Map<Long, Long> formattedQueryCount = Maps.newLinkedHashMap();
for (List<String> hit : orgQueryCount) {
@@ -894,20 +907,58 @@ private NodeInfo generateNodeInfo(long cuboidId, int dimensionCount, long cubeQu
return formattedQueryCount;
}
- public Map<Long, Map<Long, Long>> formatRollingUpCount(List<List<String>> orgRollingUpCount) {
- Map<Long, Map<Long, Long>> formattedRollingUpCount = Maps.newLinkedHashMap();
+ public Map<Long, Map<Long, Pair<Long, Long>>> formatRollingUpStats(List<List<String>> orgRollingUpCount) {
+ Map<Long, Map<Long, Pair<Long, Long>>> formattedRollingUpStats = Maps.newLinkedHashMap();
for (List<String> rollingUp : orgRollingUpCount) {
- Map<Long, Long> childMap = Maps.newLinkedHashMap();
- childMap.put(Long.parseLong(rollingUp.get(1)), (long) Double.parseDouble(rollingUp.get(2)));
- formattedRollingUpCount.put(Long.parseLong(rollingUp.get(0)), childMap);
- }
- return formattedRollingUpCount;
- }
-
- public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
- Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
- aclEvaluate.checkProjectAdminPermission(cube.getProject());
- return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap);
+ Map<Long, Pair<Long, Long>> childMap = Maps.newLinkedHashMap();
+ Long srcCuboid = Long.parseLong(rollingUp.get(0));
+ Long tgtCuboid = Long.parseLong(rollingUp.get(1));
+ Long rollupCount = (long) Double.parseDouble(rollingUp.get(2));
+ Long returnCount = (long) Double.parseDouble(rollingUp.get(3));
+ childMap.put(tgtCuboid, new Pair<>(rollupCount, returnCount));
+ formattedRollingUpStats.put(srcCuboid, childMap);
+ }
+ return formattedRollingUpStats;
+ }
+
+ public Map<Long, Long> getCuboidHitFrequency(String cubeName, boolean isCuboidSource) {
+ String cuboidColumn = isCuboidSource ? QueryCubePropertyEnum.CUBOID_SOURCE.toString()
+ : QueryCubePropertyEnum.CUBOID_TARGET.toString();
+ String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
+ String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
+ String sql = "select " + cuboidColumn + ", sum(" + hitMeasure + ")" //
+ + " from " + table//
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "'" //
+ + " group by " + cuboidColumn;
+ List<List<String>> orgHitFrequency = queryService.querySystemCube(sql).getResults();
+ return formatQueryCount(orgHitFrequency);
+ }
+
+ public Map<Long, Map<Long, Pair<Long, Long>>> getCuboidRollingUpStats(String cubeName) {
+ String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
+ String cuboidTgt = QueryCubePropertyEnum.CUBOID_TARGET.toString();
+ String aggCount = QueryCubePropertyEnum.AGGR_COUNT.toString();
+ String returnCount = QueryCubePropertyEnum.RETURN_COUNT.toString();
+ String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
+ String sql = "select " + cuboidSource + ", " + cuboidTgt + ", avg(" + aggCount + "), avg(" + returnCount + ")"//
+ + " from " + table //
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " //
+ + " group by " + cuboidSource + ", " + cuboidTgt;
+ List<List<String>> orgRollingUpCount = queryService.querySystemCube(sql).getResults();
+ return formatRollingUpStats(orgRollingUpCount);
+ }
+
+ public Map<Long, Long> getCuboidQueryMatchCount(String cubeName) {
+ String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
+ String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
+ String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
+ String sql = "select " + cuboidSource + ", sum(" + hitMeasure + ")" //
+ + " from " + table //
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "'" //
+ + " and " + QueryCubePropertyEnum.IF_MATCH.toString() + " = true" //
+ + " group by " + cuboidSource;
+ List<List<String>> orgMatchHitFrequency = queryService.querySystemCube(sql).getResults();
+ return formatQueryCount(orgMatchHitFrequency);
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java b/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
index ec395e024e..39102456eb 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
@@ -58,9 +58,12 @@
}
private enum QueryDimensionEnum {
- PROJECT(QueryPropertyEnum.PROJECT.toString()), CUBE(QueryPropertyEnum.REALIZATION.toString()), DAY(
- TimePropertyEnum.DAY_DATE.toString()), WEEK(
- TimePropertyEnum.WEEK_BEGIN_DATE.toString()), MONTH(TimePropertyEnum.MONTH.toString());
+ PROJECT(QueryPropertyEnum.PROJECT.toString()), //
+ CUBE(QueryPropertyEnum.REALIZATION.toString()), //
+ DAY(TimePropertyEnum.DAY_DATE.toString()), //
+ WEEK(TimePropertyEnum.WEEK_BEGIN_DATE.toString()), //
+ MONTH(TimePropertyEnum.MONTH.toString());
+
private final String sql;
QueryDimensionEnum(String sql) {
@@ -73,9 +76,12 @@ public String toSQL() {
};
private enum JobDimensionEnum {
- PROJECT(JobPropertyEnum.PROJECT.toString()), CUBE(JobPropertyEnum.CUBE.toString()), DAY(
- TimePropertyEnum.DAY_DATE.toString()), WEEK(
- TimePropertyEnum.WEEK_BEGIN_DATE.toString()), MONTH(TimePropertyEnum.MONTH.toString());
+ PROJECT(JobPropertyEnum.PROJECT.toString()), //
+ CUBE(JobPropertyEnum.CUBE.toString()), //
+ DAY(TimePropertyEnum.DAY_DATE.toString()), //
+ WEEK(TimePropertyEnum.WEEK_BEGIN_DATE.toString()), //
+ MONTH(TimePropertyEnum.MONTH.toString());
+
private final String sql;
JobDimensionEnum(String sql) {
@@ -88,10 +94,10 @@ public String toSQL() {
};
private enum QueryMetricEnum {
- QUERY_COUNT("count(*)"), AVG_QUERY_LATENCY("sum(" + QueryPropertyEnum.TIME_COST.toString() + ")/(count("
- + QueryPropertyEnum.TIME_COST.toString() + "))"), MAX_QUERY_LATENCY(
- "max(" + QueryPropertyEnum.TIME_COST.toString() + ")"), MIN_QUERY_LATENCY(
- "min(" + QueryPropertyEnum.TIME_COST.toString() + ")");
+ QUERY_COUNT("count(*)"), //
+ AVG_QUERY_LATENCY("avg(" + QueryPropertyEnum.TIME_COST.toString() + ")"), //
+ MAX_QUERY_LATENCY("max(" + QueryPropertyEnum.TIME_COST.toString() + ")"), //
+ MIN_QUERY_LATENCY("min(" + QueryPropertyEnum.TIME_COST.toString() + ")");
private final String sql;
@@ -105,10 +111,10 @@ public String toSQL() {
}
private enum JobMetricEnum {
- JOB_COUNT("count(*)"), AVG_JOB_BUILD_TIME("sum(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")/count("
- + JobPropertyEnum.PER_BYTES_TIME_COST + ")"), MAX_JOB_BUILD_TIME(
- "max(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"), MIN_JOB_BUILD_TIME(
- "min(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")");
+ JOB_COUNT("count(*)"), //
+ AVG_JOB_BUILD_TIME("avg(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"), //
+ MAX_JOB_BUILD_TIME("max(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"), //
+ MIN_JOB_BUILD_TIME("min(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")");
private final String sql;
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 02bb804a88..da4fcb6430 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -93,6 +93,7 @@
import org.apache.kylin.metadata.querymeta.TableMeta;
import org.apache.kylin.metadata.querymeta.TableMetaWithType;
import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metrics.MetricsManager;
import org.apache.kylin.query.QueryConnection;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.util.PushDownUtil;
@@ -352,6 +353,13 @@ public void logQuery(final String queryId, final SQLRequest request, final SQLRe
logger.info(stringBuilder.toString());
}
+ public SQLResponse querySystemCube(String sql) {
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ sqlRequest.setSql(sql);
+ return doQueryWithCache(sqlRequest, false);
+ }
+
public SQLResponse doQueryWithCache(SQLRequest sqlRequest) {
long t = System.currentTimeMillis();
aclEvaluate.checkProjectReadPermission(sqlRequest.getProject());
@@ -375,7 +383,8 @@ public SQLResponse doQueryWithCache(SQLRequest sqlRequest, boolean isQueryInspec
// project not found
ProjectManager mgr = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
if (mgr.getProject(sqlRequest.getProject()) == null) {
- throw new BadRequestException(String.format(Locale.ROOT, msg.getPROJECT_NOT_FOUND(), sqlRequest.getProject()));
+ throw new BadRequestException(
+ String.format(Locale.ROOT, msg.getPROJECT_NOT_FOUND(), sqlRequest.getProject()));
}
if (StringUtils.isBlank(sqlRequest.getSql())) {
throw new BadRequestException(msg.getNULL_EMPTY_SQL());
@@ -1020,8 +1029,7 @@ protected String makeErrorMsgUserFriendly(Throwable e) {
}
private SQLResponse getPrepareOnlySqlResponse(String projectName, String correctedSql, Connection conn,
- Boolean isPushDown,
- List<List<String>> results, List<SelectedColumnMeta> columnMetas) throws SQLException {
+ Boolean isPushDown, List<List<String>> results, List<SelectedColumnMeta> columnMetas) throws SQLException {
CalcitePrepareImpl.KYLIN_ONLY_PREPARE.set(true);
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services