You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/11/16 04:15:26 UTC
[2/2] incubator-kylin git commit: minor: fix bugs when merging codes
minor: fix bugs when merging codes
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/71e6bed3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/71e6bed3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/71e6bed3
Branch: refs/heads/2.x-staging
Commit: 71e6bed33bd5d71113ab609515060efe0b49a520
Parents: 8e8187b
Author: honma <ho...@ebay.com>
Authored: Mon Nov 16 11:19:54 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Nov 16 11:19:54 2015 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/cube/CubeSegment.java | 2 ++
.../org/apache/kylin/engine/mr/BatchCubingJobBuilder.java | 10 +++++-----
.../org/apache/kylin/engine/mr/BatchMergeJobBuilder.java | 6 ++++--
3 files changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/71e6bed3/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 076bd14..62df1e9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -28,9 +28,11 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.ShardingHash;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.IDictionaryAware;
import org.apache.kylin.metadata.model.IBuildable;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/71e6bed3/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
index b5a7272..12cac94 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
@@ -44,13 +44,13 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
Preconditions.checkArgument(!newSegment.isEnableSharding(), "V1 job engine does not support building sharded cubes");
this.inputSide = MRUtil.getBatchCubingInputSide(seg);
- this.outputSide = MRUtil.getBatchCubingOutputSide((CubeSegment)seg);
+ this.outputSide = MRUtil.getBatchCubingOutputSide((CubeSegment) seg);
}
public CubingJob build() {
logger.info("MR_V1 new job to BUILD segment " + seg);
- final CubingJob result = CubingJob.createBuildJob(seg, submitter, config);
+ final CubingJob result = CubingJob.createBuildJob((CubeSegment) seg, submitter, config);
final String jobId = result.getId();
final String cuboidRootPath = getCuboidRootPath(jobId);
@@ -62,7 +62,7 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
result.addTask(createBuildDictionaryStep(jobId));
// Phase 3: Build Cube
- RowKeyDesc rowKeyDesc = ((CubeSegment)seg).getCubeDesc().getRowkey();
+ RowKeyDesc rowKeyDesc = ((CubeSegment) seg).getCubeDesc().getRowkey();
final int groupRowkeyColumnsCount = rowKeyDesc.getNCuboidBuildLevels();
final int totalRowkeyColumnsCount = rowKeyDesc.getRowKeyColumns().length;
final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount);
@@ -88,7 +88,7 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
MapReduceExecutable baseCuboidStep = new MapReduceExecutable();
StringBuilder cmd = new StringBuilder();
- appendMapReduceParameters(cmd, ((CubeSegment)seg).getCubeDesc().getModel());
+ appendMapReduceParameters(cmd, ((CubeSegment) seg).getCubeDesc().getModel());
baseCuboidStep.setName(ExecutableConstants.STEP_NAME_BUILD_BASE_CUBOID);
@@ -112,7 +112,7 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
ndCuboidStep.setName(ExecutableConstants.STEP_NAME_BUILD_N_D_CUBOID + " : " + dimNum + "-Dimension");
StringBuilder cmd = new StringBuilder();
- appendMapReduceParameters(cmd, ((CubeSegment)seg).getCubeDesc().getModel());
+ appendMapReduceParameters(cmd, ((CubeSegment) seg).getCubeDesc().getModel());
appendExecCmdParameters(cmd, "cubename", seg.getRealization().getName());
appendExecCmdParameters(cmd, "segmentname", seg.getName());
appendExecCmdParameters(cmd, "input", cuboidOutputTempPath[totalRowkeyColumnCount - dimNum - 1]);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/71e6bed3/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
index 1282e61..831aa9d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder.java
@@ -42,13 +42,15 @@ public class BatchMergeJobBuilder extends JobBuilderSupport {
Preconditions.checkArgument(!mergeSegment.isEnableSharding(), "V1 job engine does not support merging sharded cubes");
- this.outputSide = MRUtil.getBatchMergeOutputSide(seg);
+ this.outputSide = MRUtil.getBatchMergeOutputSide(mergeSegment);
}
public CubingJob build() {
logger.info("MR_V1 new job to MERGE segment " + seg);
- final CubingJob result = CubingJob.createMergeJob(seg, submitter, config);
+ final CubeSegment cubeSegment = (CubeSegment)seg;
+
+ final CubingJob result = CubingJob.createMergeJob(cubeSegment, submitter, config);
final String jobId = result.getId();
final String cuboidRootPath = getCuboidRootPath(jobId);