You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/09/23 23:36:11 UTC
[19/20] kylin git commit: Merge commit
'658353ad57ebb30e6e6a740769a85472354212d9'
Merge commit '658353ad57ebb30e6e6a740769a85472354212d9'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/452f4685
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/452f4685
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/452f4685
Branch: refs/heads/master
Commit: 452f468533de169f6fe5aa24717367281350376d
Parents: b15164a 658353a
Author: Li Yang <li...@apache.org>
Authored: Sat Sep 23 21:45:52 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Sep 23 21:45:52 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 32 +
.../org/apache/kylin/common/util/JsonUtil.java | 3 +-
.../common/util/MemoryBudgetController.java | 2 +-
.../main/resources/kylin-defaults.properties | 9 +
.../org/apache/kylin/cube/CubeInstance.java | 9 +-
.../java/org/apache/kylin/cube/CubeManager.java | 34 +-
.../java/org/apache/kylin/cube/CubeUpdate.java | 11 +
.../kylin/cube/cuboid/TreeCuboidScheduler.java | 3 +-
.../algorithm/AbstractRecommendAlgorithm.java | 82 +
.../cube/cuboid/algorithm/BPUSCalculator.java | 165 +
.../cube/cuboid/algorithm/BenefitPolicy.java | 41 +
.../cuboid/algorithm/CuboidBenefitModel.java | 210 +
.../algorithm/CuboidRecommendAlgorithm.java | 49 +
.../cuboid/algorithm/CuboidRecommender.java | 163 +
.../cube/cuboid/algorithm/CuboidStats.java | 291 ++
.../cube/cuboid/algorithm/CuboidStatsUtil.java | 206 +
.../cube/cuboid/algorithm/PBPUSCalculator.java | 53 +
.../cube/cuboid/algorithm/SPBPUSCalculator.java | 41 +
.../algorithm/generic/BitsChromosome.java | 133 +
.../generic/CombinedStoppingCondition.java | 43 +
.../cuboid/algorithm/generic/CuboidEncoder.java | 44 +
.../algorithm/generic/GeneticAlgorithm.java | 283 ++
.../generic/RouletteWheelSelection.java | 61 +
.../algorithm/generic/lib/BitsMutation.java | 59 +
.../algorithm/generic/lib/Chromosome.java | 104 +
.../lib/ChromosomeMismatchException.java | 48 +
.../algorithm/generic/lib/ChromosomePair.java | 67 +
.../algorithm/generic/lib/CrossoverPolicy.java | 37 +
.../generic/lib/ElitisticListPopulation.java | 117 +
.../cuboid/algorithm/generic/lib/Fitness.java | 33 +
.../generic/lib/FixedGenerationCount.java | 71 +
.../algorithm/generic/lib/ListPopulation.java | 223 +
.../algorithm/generic/lib/MutationPolicy.java | 34 +
.../generic/lib/OnePointCrossover.java | 126 +
.../algorithm/generic/lib/Population.java | 59 +
.../algorithm/generic/lib/SelectionPolicy.java | 33 +
.../generic/lib/StoppingCondition.java | 34 +
.../generic/lib/TournamentSelection.java | 114 +
.../algorithm/greedy/GreedyAlgorithm.java | 170 +
.../cube/inmemcubing/ConcurrentDiskStore.java | 2 +-
.../kylin/cube/inmemcubing/MemDiskStore.java | 2 +-
.../kylin/cube/model/AggregationGroup.java | 70 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 4 +-
.../cube/model/TooManyCuboidException.java | 1 +
.../kylin/cube/AggregationGroupRuleTest.java | 10 +-
.../org/apache/kylin/cube/CubeDescTest.java | 4 +-
.../org/apache/kylin/cube/CubeInstanceTest.java | 57 +
.../org/apache/kylin/cube/CubeManagerTest.java | 3 +-
.../cuboid/algorithm/AlgorithmTestBase.java | 310 ++
.../cuboid/algorithm/CuboidStatsUtilTest.java | 170 +
.../cuboid/algorithm/GeneticAlgorithmTest.java | 57 +
.../cuboid/algorithm/GreedyAlgorithmTest.java | 58 +
.../kylin/cube/project/ProjectManagerTest.java | 5 +-
.../src/test/resources/learn_kylin_cube2.json | 84 +
core-cube/src/test/resources/statistics.txt | 4092 ++++++++++++++++++
.../apache/kylin/dict/AppendTrieDictionary.java | 37 +-
.../apache/kylin/dict/DictionaryManager.java | 10 +
.../kylin/dict/GlobalDictionaryBuilder.java | 8 +-
.../global/AppendTrieDictionaryBuilder.java | 15 +-
.../kylin/dict/global/GlobalDictHDFSStore.java | 80 +-
.../kylin/dict/global/GlobalDictMetadata.java | 1 -
.../kylin/dict/global/GlobalDictStore.java | 5 +-
.../global/SegmentAppendTrieDictBuilder.java | 73 +
.../apache/kylin/dict/NumberDictionaryTest.java | 8 +-
.../dict/global/AppendTrieDictionaryTest.java | 3 +-
.../kylin/measure/bitmap/BitmapMeasureType.java | 21 +-
.../kylin/source/datagen/DataGenTest.java | 4 +-
.../kylin/engine/mr/common/CubeStatsReader.java | 4 +
.../engine/mr/common/CuboidRecommenderUtil.java | 106 +
.../engine/mr/common/CuboidStatsReaderUtil.java | 135 +
.../mr/common/StatisticsDecisionUtil.java | 114 +
.../engine/mr/steps/SaveStatisticsStep.java | 55 +-
.../mr/steps/NumberDictionaryForestTest.java | 17 +-
.../localmeta/cube_desc/ci_inner_join_cube.json | 2 +-
pom.xml | 5 +-
.../apache/kylin/rest/service/QueryService.java | 9 +-
.../storage/hbase/steps/CreateHTableJob.java | 6 +-
webapp/app/js/model/cubeConfig.js | 3 +-
.../cubeDesigner/advanced_settings.html | 13 +-
webapp/app/partials/query/query.html | 2 +-
80 files changed, 8717 insertions(+), 245 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/core-common/src/main/resources/kylin-defaults.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --cc core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 425e32f,0ba7f21..33dc312
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@@ -314,9 -314,9 +314,6 @@@ public class CubeManager implements IRe
// delete cube from project
ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName);
-- if (listener != null)
-- listener.afterCubeDelete(cube);
--
return cube;
}
@@@ -332,9 -332,9 +329,6 @@@
updateCubeWithRetry(new CubeUpdate(cube), 0);
ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName, owner);
-- if (listener != null)
-- listener.afterCubeCreate(cube);
--
return cube;
}
@@@ -348,18 -348,18 +342,11 @@@
ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cube.getName(), projectName,
owner);
-- if (listener != null)
-- listener.afterCubeCreate(cube);
--
return cube;
}
public CubeInstance updateCube(CubeUpdate update) throws IOException {
CubeInstance cube = updateCubeWithRetry(update, 0);
--
-- if (listener != null)
-- listener.afterCubeUpdate(cube);
--
return cube;
}
@@@ -843,23 -847,23 +834,6 @@@
return getCube(name);
}
-- // ============================================================================
--
-- public interface CubeChangeListener {
-- void afterCubeCreate(CubeInstance cube);
--
-- void afterCubeUpdate(CubeInstance cube);
--
-- void afterCubeDelete(CubeInstance cube);
-- }
--
-- private CubeChangeListener listener;
--
-- public void setCubeChangeListener(CubeChangeListener listener) {
-- this.listener = listener;
-- }
--
--
/**
* Calculate the holes (gaps) in segments.
* @param cubeName
http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --cc core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index d473858,74ac715..d976c79
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@@ -300,42 -301,48 +301,51 @@@ public class AggregationGroup implement
public long calculateCuboidCombination() {
long combination = 1;
- if (this.getDimCap() > 0) {
- CuboidScheduler cuboidScheduler = cubeDesc.getInitialCuboidScheduler();
- combination = cuboidScheduler.calculateCuboidsForAggGroup(this).size();
- } else {
- Set<String> includeDims = new TreeSet<>(Arrays.asList(includes));
- Set<String> mandatoryDims = new TreeSet<>(Arrays.asList(selectRule.mandatoryDims));
-
- Set<String> hierarchyDims = new TreeSet<>();
- for (String[] ss : selectRule.hierarchyDims) {
- hierarchyDims.addAll(Arrays.asList(ss));
- combination = combination * (ss.length + 1);
- }
-
- Set<String> jointDims = new TreeSet<>();
- for (String[] ss : selectRule.jointDims) {
- jointDims.addAll(Arrays.asList(ss));
- combination = combination * 2;
- }
-
- Set<String> normalDims = new TreeSet<>();
- normalDims.addAll(includeDims);
- normalDims.removeAll(mandatoryDims);
- normalDims.removeAll(hierarchyDims);
- normalDims.removeAll(jointDims);
-
- combination = combination * (1L << normalDims.size());
-
- if (cubeDesc.getConfig().getCubeAggrGroupIsMandatoryOnlyValid() && !mandatoryDims.isEmpty()) {
- combination += 1;
+ try {
+ if (this.getDimCap() > 0) {
+ CuboidScheduler cuboidScheduler = cubeDesc.getInitialCuboidScheduler();
+ combination = cuboidScheduler.calculateCuboidsForAggGroup(this).size();
+ } else {
+ Set<String> includeDims = new TreeSet<>(Arrays.asList(includes));
+ Set<String> mandatoryDims = new TreeSet<>(Arrays.asList(selectRule.mandatoryDims));
+
+ Set<String> hierarchyDims = new TreeSet<>();
+ for (String[] ss : selectRule.hierarchyDims) {
+ hierarchyDims.addAll(Arrays.asList(ss));
+ combination = LongMath.checkedMultiply(combination, (ss.length + 1));
+ }
+
+ Set<String> jointDims = new TreeSet<>();
+ for (String[] ss : selectRule.jointDims) {
+ jointDims.addAll(Arrays.asList(ss));
+ }
+ combination = LongMath.checkedMultiply(combination, (1L << selectRule.jointDims.length));
+
+ Set<String> normalDims = new TreeSet<>();
+ normalDims.addAll(includeDims);
+ normalDims.removeAll(mandatoryDims);
+ normalDims.removeAll(hierarchyDims);
+ normalDims.removeAll(jointDims);
+
+ combination = LongMath.checkedMultiply(combination, (1L << normalDims.size()));
+
+ if (cubeDesc.getConfig().getCubeAggrGroupIsMandatoryOnlyValid() && !mandatoryDims.isEmpty()) {
+ combination += 1;
+ }
+ combination -= 1; // not include cuboid 0
}
- combination -= 1; // not include cuboid 0
+
+ if (combination < 0)
+ throw new ArithmeticException();
+
+ } catch (ArithmeticException ae) {
+ // long overflow, give max value
+ combination = Long.MAX_VALUE;
}
+ if (combination < 0) { // overflow
+ combination = Long.MAX_VALUE - 1;
+ }
return combination;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/452f4685/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
----------------------------------------------------------------------