You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/06/02 14:19:54 UTC
incubator-kylin git commit: KYLIN-807 fix a bug and add a small test
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8.0 da17eec63 -> f2d6c13c5
KYLIN-807 fix a bug and add a small test
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f2d6c13c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f2d6c13c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f2d6c13c
Branch: refs/heads/0.8.0
Commit: f2d6c13c5319b2d08d11955398c259aced5606aa
Parents: da17eec
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jun 2 20:19:47 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Jun 2 20:19:47 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../kylin/job/streaming/CubeStreamBuilder.java | 5 +--
.../cubev2/FactDistinctColumnsReducerTest.java | 36 ++++++++++++++++++++
3 files changed, 38 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2d6c13c/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 44153e9..1de8fa4 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -240,7 +240,7 @@ public class CubeManager implements IRealizationProvider {
if (cube == null)
throw new IllegalStateException();
- logger.info("Updating cube instance '" + cube.getName());
+ logger.info("Updating cube instance '" + cube.getName() + "'");
if (toAddSegs != null && toAddSegs.size() > 0)
cube.getSegments().addAll(toAddSegs);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2d6c13c/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
index 7c3eee8..2831caa 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamBuilder.java
@@ -99,10 +99,7 @@ public class CubeStreamBuilder extends StreamBuilder {
final Map<Long, HyperLogLogPlusCounter> samplingResult = sampling(cubeInstance.getDescriptor(), parsedStreamMessages);
final Configuration conf = HadoopUtil.getCurrentConfiguration();
- final Path outputPath = new Path("file:///tmp/kylin/cuboidstatistics/" + UUID.randomUUID().toString());
- if (!FileSystem.getLocal(conf).exists(outputPath)) {
- FileSystem.getLocal(conf).create(outputPath);
- }
+ final Path outputPath = new Path("file:///tmp/cuboidstatistics/" + UUID.randomUUID().toString());
FactDistinctColumnsReducer.writeCuboidStatistics(conf, outputPath, samplingResult, 100);
ResourceStore.getStore(kylinConfig).putResource(cubeSegment.getStatisticsResourcePath(), FileSystem.getLocal(conf).open(new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION)), 0);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f2d6c13c/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/FactDistinctColumnsReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/FactDistinctColumnsReducerTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/FactDistinctColumnsReducerTest.java
new file mode 100644
index 0000000..2e2a945
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/FactDistinctColumnsReducerTest.java
@@ -0,0 +1,36 @@
+package org.apache.kylin.job.hadoop.cubev2;
+
+import com.google.common.collect.Maps;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.job.hadoop.cube.FactDistinctColumnsReducer;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ */
+public class FactDistinctColumnsReducerTest {
+
+
+ @Test
+ public void testWriteCuboidStatistics() throws IOException {
+
+ final Configuration conf = HadoopUtil.getCurrentConfiguration();
+ final Path outputPath = new Path("file:///tmp/kylin/cuboidstatistics/" + UUID.randomUUID().toString());
+ if (!FileSystem.getLocal(conf).exists(outputPath)) {
+// FileSystem.getLocal(conf).create(outputPath);
+ }
+
+ System.out.println(outputPath);
+ Map<Long, HyperLogLogPlusCounter> cuboidHLLMap = Maps.newHashMap();
+ FactDistinctColumnsReducer.writeCuboidStatistics(conf, outputPath, cuboidHLLMap, 100);
+
+
+ }
+}