You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/01/23 09:31:58 UTC
[11/50] [abbrv] incubator-kylin git commit: merge segment test pass
merge segment test pass
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/58a3239b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/58a3239b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/58a3239b
Branch: refs/heads/inverted-index
Commit: 58a3239b5110d69dda3d23a8d93d92675ac9e5c4
Parents: 7c61ca9
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Jan 22 14:32:16 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jan 22 14:32:16 2015 +0800
----------------------------------------------------------------------
.../com/kylinolap/job/constant/ExecutableConstants.java | 1 +
.../main/java/com/kylinolap/job/cube/CubingJobBuilder.java | 6 +++---
.../java/com/kylinolap/job/cube/MergeDictionaryStep.java | 3 ---
.../com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java | 9 +++++++--
4 files changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/58a3239b/job/src/main/java/com/kylinolap/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/constant/ExecutableConstants.java b/job/src/main/java/com/kylinolap/job/constant/ExecutableConstants.java
index adf6122..93e81c9 100644
--- a/job/src/main/java/com/kylinolap/job/constant/ExecutableConstants.java
+++ b/job/src/main/java/com/kylinolap/job/constant/ExecutableConstants.java
@@ -34,6 +34,7 @@ public final class ExecutableConstants {
public static final String STEP_NAME_CREATE_HBASE_TABLE = "Create HTable";
public static final String STEP_NAME_CONVERT_CUBOID_TO_HFILE = "Convert Cuboid Data to HFile";
public static final String STEP_NAME_BULK_LOAD_HFILE = "Load HFile to HBase Table";
+ public static final String STEP_NAME_MERGE_DICTIONARY = "Merge Cuboid Dictionary";
public static final String STEP_NAME_MERGE_CUBOID = "Merge Cuboid Data";
public static final String STEP_NAME_UPDATE_CUBE_INFO = "Update Cube Info";
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/58a3239b/job/src/main/java/com/kylinolap/job/cube/CubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/cube/CubingJobBuilder.java b/job/src/main/java/com/kylinolap/job/cube/CubingJobBuilder.java
index b94360a..c89a7ef 100644
--- a/job/src/main/java/com/kylinolap/job/cube/CubingJobBuilder.java
+++ b/job/src/main/java/com/kylinolap/job/cube/CubingJobBuilder.java
@@ -350,9 +350,9 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
return updateCubeInfoStep;
}
- private UpdateCubeInfoAfterMergeStep createMergeDictionaryStep(List<String> mergingSegmentIds) {
- UpdateCubeInfoAfterMergeStep result = new UpdateCubeInfoAfterMergeStep();
- result.setName(ExecutableConstants.STEP_NAME_UPDATE_CUBE_INFO);
+ private MergeDictionaryStep createMergeDictionaryStep(List<String> mergingSegmentIds) {
+ MergeDictionaryStep result = new MergeDictionaryStep();
+ result.setName(ExecutableConstants.STEP_NAME_MERGE_DICTIONARY);
result.setCubeName(getCubeName());
result.setSegmentId(segment.getUuid());
result.setMergingSegmentIds(mergingSegmentIds);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/58a3239b/job/src/main/java/com/kylinolap/job/cube/MergeDictionaryStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/cube/MergeDictionaryStep.java b/job/src/main/java/com/kylinolap/job/cube/MergeDictionaryStep.java
index 629dc0d..c13bdc0 100644
--- a/job/src/main/java/com/kylinolap/job/cube/MergeDictionaryStep.java
+++ b/job/src/main/java/com/kylinolap/job/cube/MergeDictionaryStep.java
@@ -50,9 +50,6 @@ public class MergeDictionaryStep extends AbstractExecutable {
makeDictForNewSegment(conf, cube, newSegment, mergingSegments);
makeSnapshotForNewSegment(cube, newSegment, mergingSegments);
- cube.getSegments().add(newSegment);
- Collections.sort(cube.getSegments());
-
mgr.updateCube(cube);
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/58a3239b/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java
index 7103dcc..ed94b50 100644
--- a/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java
@@ -23,6 +23,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import com.kylinolap.common.KylinConfig;
@@ -94,7 +95,8 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
private CubeSegment findSegmentWithUuid(String jobID, CubeInstance cubeInstance) {
for (CubeSegment segment : cubeInstance.getSegments()) {
- if (segment.getLastBuildJobID().equalsIgnoreCase(jobID)) {
+ String lastBuildJobID = segment.getLastBuildJobID();
+ if (lastBuildJobID != null && lastBuildJobID.equalsIgnoreCase(jobID)) {
return segment;
}
}
@@ -121,10 +123,13 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
newKeyBuf = new byte[256];// size will auto-grow
// decide which source segment
- org.apache.hadoop.mapreduce.InputSplit inputSplit = context.getInputSplit();
+ InputSplit inputSplit = context.getInputSplit();
String filePath = ((FileSplit) inputSplit).getPath().toString();
+ System.out.println("filePath:" + filePath);
String jobID = extractJobIDFromPath(filePath);
+ System.out.println("jobID:" + jobID);
sourceCubeSegment = findSegmentWithUuid(jobID, cube);
+ System.out.println(sourceCubeSegment);
this.rowKeySplitter = new RowKeySplitter(sourceCubeSegment, 65, 255);
}