You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/12/25 11:13:01 UTC
kylin git commit: KYLIN-1245 bug fix and better log
Repository: kylin
Updated Branches:
refs/heads/2.x-staging 66d275cb3 -> 6ecdd08da
KYLIN-1245 bug fix and better log
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6ecdd08d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6ecdd08d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6ecdd08d
Branch: refs/heads/2.x-staging
Commit: 6ecdd08dab35f317b416f867f8350107f209b073
Parents: 66d275c
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Dec 25 18:12:38 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Dec 25 18:12:38 2015 +0800
----------------------------------------------------------------------
core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java | 5 ++---
.../main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java | 2 +-
.../kylin/engine/mr/steps/FactDistinctColumnsReducer.java | 2 +-
.../java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java | 2 +-
.../org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java | 4 ++--
5 files changed, 7 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6ecdd08d/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index b29f83a..67dce73 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -41,7 +41,6 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonBackReference;
import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
import com.google.common.collect.Maps;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
@@ -265,7 +264,7 @@ public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware, I
public void setStorageLocationIdentifier(String storageLocationIdentifier) {
this.storageLocationIdentifier = storageLocationIdentifier;
}
-
+
public Map<TblColRef, Dictionary<String>> buildDictionaryMap() {
Map<TblColRef, Dictionary<String>> result = Maps.newHashMap();
for (TblColRef col : getCubeDesc().getAllColumnsNeedDictionary()) {
@@ -348,7 +347,7 @@ public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware, I
@Override
public String toString() {
- return Objects.toStringHelper(this).add("cube", cubeInstance.getName()).add("name", name).add("status", status).add("uuid", uuid).add("create_time_utc:", createTimeUTC).add("last_build_job_id", lastBuildJobID).toString();
+ return cubeInstance.getName() + "[" + name + "]";
}
public void setDictionaries(ConcurrentHashMap<String, String> dictionaries) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6ecdd08d/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
index 453e8fe..d24772c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
@@ -105,7 +105,7 @@ public class CuboidJob extends AbstractHadoopJob {
CubeInstance cube = cubeMgr.getCube(cubeName);
if (checkSkip(cubingJobId)) {
- logger.info("Skip job " + getOptionValue(OPTION_JOB_NAME) + " for " + segmentName);
+ logger.info("Skip job " + getOptionValue(OPTION_JOB_NAME) + " for " + cubeName + "[" + segmentName + "]");
return 0;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6ecdd08d/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index b63026e..65d1530 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -163,7 +163,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<Text, Text, NullWri
for (HyperLogLogPlusCounter hll : cuboidHLLMap.values()) {
grandTotal += hll.getCountEstimate();
}
- double mapperOverlapRatio = (double) totalRowsBeforeMerge / grandTotal;
+ double mapperOverlapRatio = grandTotal == 0 ? 0 : (double) totalRowsBeforeMerge / grandTotal;
writeMapperAndCuboidStatistics(context); // for human check
CuboidStatsUtil.writeCuboidStatistics(context.getConfiguration(), new Path(statisticsOutput), //
http://git-wip-us.apache.org/repos/asf/kylin/blob/6ecdd08d/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
index 6fec03c..d197252 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java
@@ -91,7 +91,7 @@ public class InMemCuboidJob extends AbstractHadoopJob {
String cubingJobId = getOptionValue(OPTION_CUBING_JOB_ID);
if (checkSkip(cubingJobId)) {
- logger.info("Skip job " + getOptionValue(OPTION_JOB_NAME) + " for " + cubeSeg.getName());
+ logger.info("Skip job " + getOptionValue(OPTION_JOB_NAME) + " for " + cubeSeg);
return 0;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6ecdd08d/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index b12063f..396955b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -100,11 +100,11 @@ public class SaveStatisticsStep extends AbstractExecutable {
} else { // the default
double threshold = kylinConf.getCubeAlgorithmAutoThreshold();
double mapperOverlapRatio = new CubeStatsReader(seg, kylinConf).getMapperOverlapRatioOfFirstBuild();
- logger.info("mapperOverlapRatio for " + seg.getName() + " is " + mapperOverlapRatio + " and threshold is " + threshold);
+ logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + " and threshold is " + threshold);
alg = mapperOverlapRatio < threshold ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER;
}
- logger.info("The cube algorithm for " + seg.getName() + " is " + alg);
+ logger.info("The cube algorithm for " + seg + " is " + alg);
CubingJob cubingJob = (CubingJob) executableManager.getJob(getCubingJobId());
cubingJob.setAlgorithm(alg);