You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/02/04 00:51:41 UTC
[38/50] [abbrv] kylin git commit: KYLIN-3209: Make optimize job
partial statistics path consistent with batch building ones
KYLIN-3209: Make optimize job partial statistics path consistent with batch building ones
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0f63fa61
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0f63fa61
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0f63fa61
Branch: refs/heads/sync
Commit: 0f63fa617ffc9604e61ba024f8dcf8acb30a54a2
Parents: 5eedd13
Author: Ma Gang <mg...@163.com>
Authored: Tue Jan 30 11:17:33 2018 +0800
Committer: Zhong <nj...@apache.org>
Committed: Tue Jan 30 11:24:41 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java | 4 +++-
.../apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0f63fa61/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
index b1e59a7..f50a4be 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
@@ -50,10 +50,12 @@ public class CubeStatsWriter {
mapperOverlapRatio);
}
+ //Be care of that the file name for partial cuboid statistics should start with BatchConstants.CFG_OUTPUT_STATISTICS,
+ //Then for later statistics merging, only files starting with BatchConstants.CFG_OUTPUT_STATISTICS will be used
public static void writePartialCuboidStatistics(Configuration conf, Path outputPath, //
Map<Long, HLLCounter> cuboidHLLMap, int samplingPercentage, int mapperNumber, double mapperOverlapRatio,
int shard) throws IOException {
- Path seqFilePath = new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME + "_" + shard);
+ Path seqFilePath = new Path(outputPath, BatchConstants.CFG_OUTPUT_STATISTICS + "_" + shard);
writeCuboidStatisticsInner(conf, seqFilePath, cuboidHLLMap, samplingPercentage, mapperNumber,
mapperOverlapRatio);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0f63fa61/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java
index 3f12b0d..434892c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java
@@ -86,7 +86,7 @@ public class MergeStatisticsWithOldStep extends AbstractExecutable {
}
Path[] statisticsFiles = HadoopUtil.getFilteredPath(hdfs, statisticsDirPath,
- BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
+ BatchConstants.CFG_OUTPUT_STATISTICS);
if (statisticsFiles == null) {
throw new IOException("fail to find the statistics file in base dir: " + statisticsDirPath);
}