You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/06/02 11:52:44 UTC
incubator-kylin git commit: KYLIN-807 bug fix
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8.0 749b53589 -> 9f320a693
KYLIN-807 bug fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/9f320a69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/9f320a69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/9f320a69
Branch: refs/heads/0.8.0
Commit: 9f320a693335c90795e5f99d9837908389aacf4d
Parents: 749b535
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jun 2 17:52:01 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Jun 2 17:52:33 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/job/constant/ExecutableConstants.java | 2 +-
.../job/hadoop/cube/FactDistinctColumnsReducer.java | 4 ++--
.../kylin/job/hadoop/cubev2/MergeStatisticsStep.java | 2 +-
.../apache/kylin/job/streaming/CubeStreamBuilder.java | 13 ++++++-------
4 files changed, 10 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9f320a69/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 91ef2e1..fdcfdbe 100644
--- a/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -57,7 +57,7 @@ public final class ExecutableConstants {
public static final String STEP_NAME_SAVE_STATISTICS = "Save Cuboid Statistics";
public static final String STEP_NAME_MERGE_CUBOID = "Merge Cuboid Data";
public static final String STEP_NAME_UPDATE_CUBE_INFO = "Update Cube Info";
- public static final String STEP_NAME_GARBAGE_COLLECTION = "Garbage collection";
+ public static final String STEP_NAME_GARBAGE_COLLECTION = "Garbage Collection";
public static final String STEP_NAME_BUILD_II = "Build Inverted Index";
public static final String STEP_NAME_CONVERT_II_TO_HFILE = "Convert Inverted Index Data to HFile";
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9f320a69/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
index 62bbd74..219953f 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
@@ -137,7 +137,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
//output the hll info;
if (collectStatistics) {
writeMapperAndCuboidStatistics(context); // for human check
- writeCuboidStatistics(context.getConfiguration(), statisticsOutput, cuboidHLLMap, SAMPING_PERCENTAGE); // for CreateHTableJob
+ writeCuboidStatistics(context.getConfiguration(), new Path(statisticsOutput), cuboidHLLMap, SAMPING_PERCENTAGE); // for CreateHTableJob
}
}
@@ -193,7 +193,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
}
- public static void writeCuboidStatistics(Configuration conf, String outputPath, Map<Long, HyperLogLogPlusCounter> cuboidHLLMap, int samplingPercentage) throws IOException {
+ public static void writeCuboidStatistics(Configuration conf, Path outputPath, Map<Long, HyperLogLogPlusCounter> cuboidHLLMap, int samplingPercentage) throws IOException {
Path seqFilePath = new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
SequenceFile.Writer writer = SequenceFile.createWriter(conf,
SequenceFile.Writer.file(seqFilePath), SequenceFile.Writer.keyClass(LongWritable.class),
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9f320a69/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeStatisticsStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeStatisticsStep.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeStatisticsStep.java
index 64ea14c..35cb2a2 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeStatisticsStep.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MergeStatisticsStep.java
@@ -124,7 +124,7 @@ public class MergeStatisticsStep extends AbstractExecutable {
}
}
averageSamplingPercentage = averageSamplingPercentage / this.getMergingSegmentIds().size();
- FactDistinctColumnsReducer.writeCuboidStatistics(conf, getMergedStatisticsPath(), cuboidHLLMap, averageSamplingPercentage);
+ FactDistinctColumnsReducer.writeCuboidStatistics(conf, new Path(getMergedStatisticsPath()), cuboidHLLMap, averageSamplingPercentage);
Path statisticsFilePath = new Path(getMergedStatisticsPath(), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
FileSystem fs = statisticsFilePath.getFileSystem(conf);
FSDataInputStream is = fs.open(statisticsFilePath);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9f320a69/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
index c62edfe..7c3eee8 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
@@ -99,9 +99,12 @@ public class CubeStreamBuilder extends StreamBuilder {
final Map<Long, HyperLogLogPlusCounter> samplingResult = sampling(cubeInstance.getDescriptor(), parsedStreamMessages);
final Configuration conf = HadoopUtil.getCurrentConfiguration();
- final String outputPath = "/tmp/kylin/cuboidstatistics/" + UUID.randomUUID().toString();
+ final Path outputPath = new Path("file:///tmp/kylin/cuboidstatistics/" + UUID.randomUUID().toString());
+ if (!FileSystem.getLocal(conf).exists(outputPath)) {
+ FileSystem.getLocal(conf).create(outputPath);
+ }
FactDistinctColumnsReducer.writeCuboidStatistics(conf, outputPath, samplingResult, 100);
- ResourceStore.getStore(kylinConfig).putResource(cubeSegment.getStatisticsResourcePath(), FileSystem.get(conf).open(new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION)), 0);
+ ResourceStore.getStore(kylinConfig).putResource(cubeSegment.getStatisticsResourcePath(), FileSystem.getLocal(conf).open(new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION)), 0);
final Map<TblColRef, Dictionary<?>> dictionaryMap = buildDictionary(getTblColRefMap(cubeInstance), parsedStreamMessages);
writeDictionary(cubeSegment, dictionaryMap, startOffset, endOffset);
@@ -336,11 +339,7 @@ public class CubeStreamBuilder extends StreamBuilder {
//TODO: should we use cubeManager.promoteNewlyBuiltSegments?
private void commitSegment(CubeSegment cubeSegment) throws IOException {
cubeSegment.setStatus(SegmentStatusEnum.READY);
-
- CubeInstance cube = CubeManager.getInstance(kylinConfig).reloadCubeLocal(cubeSegment.getCubeInstance().getName());
- cube.getSegments().add(cubeSegment);
- Collections.sort(cube.getSegments());
- CubeManager.getInstance(kylinConfig).updateCube(cube, Lists.newArrayList(cubeSegment), null, null, null);
+ CubeManager.getInstance(kylinConfig).updateCube(cubeSegment.getCubeInstance(), Lists.newArrayList(cubeSegment), null, null, null);
}
private List<Long> getAllCuboidIds(CubeDesc cubeDesc) {