You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/06/22 11:07:49 UTC
[1/2] kylin git commit: refactor: more checks on finding segment by id
Repository: kylin
Updated Branches:
refs/heads/master 9b8a9899b -> 91d6798d4
refactor: more checks on finding segment by id
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c736ee5f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c736ee5f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c736ee5f
Branch: refs/heads/master
Commit: c736ee5fe8009c28f5e016d24c5f113940bc11cf
Parents: 9b8a989
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Jun 22 19:06:17 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Jun 22 19:06:41 2016 +0800
----------------------------------------------------------------------
.../engine/mr/steps/SaveStatisticsStep.java | 48 ++++++++++++++++++--
1 file changed, 43 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c736ee5f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 435fb13..7cc9dc3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -21,6 +21,9 @@ package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
import java.util.Random;
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -43,9 +46,11 @@ import org.apache.kylin.metadata.model.MeasureDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+
/**
* Save the cube segment statistic to Kylin metadata store
- *
*/
public class SaveStatisticsStep extends AbstractExecutable {
@@ -55,12 +60,45 @@ public class SaveStatisticsStep extends AbstractExecutable {
super();
}
+ private CubeSegment findSegment(ExecutableContext context, String cubeName, String segmentId) {
+ final CubeManager mgr = CubeManager.getInstance(context.getConfig());
+ final CubeInstance cube = mgr.getCube(cubeName);
+
+ if (cube == null) {
+ String cubeList = StringUtils.join(Iterables.transform(mgr.listAllCubes(), new Function<CubeInstance, String>() {
+ @Nullable
+ @Override
+ public String apply(@Nullable CubeInstance input) {
+ return input.getName();
+ }
+ }).iterator(), ",");
+
+ logger.info("target cube name: {}, cube list: {}", cubeName, cubeList);
+ throw new IllegalStateException();
+ }
+
+
+ final CubeSegment newSegment = cube.getSegmentById(segmentId);
+
+ if (newSegment == null) {
+ String segmentList = StringUtils.join(Iterables.transform(cube.getSegments(), new Function<CubeSegment, String>() {
+ @Nullable
+ @Override
+ public String apply(@Nullable CubeSegment input) {
+ return input.getUuid();
+ }
+ }).iterator(), ",");
+
+ logger.info("target segment id: {}, segment list: {}", segmentId, segmentList);
+ throw new IllegalStateException();
+ }
+ return newSegment;
+ }
+
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- final CubeManager mgr = CubeManager.getInstance(context.getConfig());
- final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
- final CubeSegment newSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
- KylinConfig kylinConf = cube.getConfig();
+ CubeSegment newSegment = findSegment(context, CubingExecutableUtil.getCubeName(this.getParams()), CubingExecutableUtil.getSegmentId(this.getParams()));
+ KylinConfig kylinConf = newSegment.getConfig();
ResourceStore rs = ResourceStore.getStore(kylinConf);
try {
[2/2] kylin git commit: fix compile
Posted by ma...@apache.org.
fix compile
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/91d6798d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/91d6798d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/91d6798d
Branch: refs/heads/master
Commit: 91d6798d47db0e4c3f20578d8d81f4390b082fdb
Parents: c736ee5
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Jun 22 19:06:35 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Jun 22 19:07:24 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/gridtable/GTScanRequest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/91d6798d/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 593469a..d520efa 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -63,7 +63,7 @@ public class GTScanRequest {
this(info, ranges, dimensions, aggrGroupBy, aggrMetrics, aggrMetricsFuncs, filterPushDown, true, 0);
}
- private GTScanRequest(GTInfo info, List<GTScanRange> ranges, ImmutableBitSet dimensions, ImmutableBitSet aggrGroupBy, //
+ public GTScanRequest(GTInfo info, List<GTScanRange> ranges, ImmutableBitSet dimensions, ImmutableBitSet aggrGroupBy, //
ImmutableBitSet aggrMetrics, String[] aggrMetricsFuncs, TupleFilter filterPushDown, boolean allowPreAggregation, double aggrCacheGB) {
this.info = info;
if (ranges == null) {