You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/09/21 11:10:24 UTC
[1/2] incubator-kylin git commit: Fix a bug in
MapContextGTRecordWriter
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-943 0e978308d -> e07cdf3d0
Fix a bug in MapContextGTRecordWriter
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/cc660bf0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/cc660bf0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/cc660bf0
Branch: refs/heads/KYLIN-943
Commit: cc660bf0942deaf045eff1d077b2c5a77ed33ae2
Parents: 0e97830
Author: shaofengshi <sh...@apache.org>
Authored: Mon Sep 21 10:29:07 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Sep 21 10:29:07 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc660bf0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
index e7dbaa1..7bb2e16 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
@@ -49,12 +49,13 @@ public class MapContextGTRecordWriter implements ICuboidWriter {
public void write(long cuboidId, GTRecord record) throws IOException {
if (lastCuboidId == null || !lastCuboidId.equals(cuboidId)) {
- // output another cuboid
- initVariables(cuboidId);
if (lastCuboidId != null) {
logger.info("Cuboid " + lastCuboidId + " has " + cuboidRowCount + " rows");
cuboidRowCount = 0;
}
+ // output another cuboid
+ initVariables(cuboidId);
+ lastCuboidId = cuboidId;
}
cuboidRowCount++;
[2/2] incubator-kylin git commit: KYLIN-943 no buildAndMerge anymore
Posted by sh...@apache.org.
KYLIN-943 no buildAndMerge anymore
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e07cdf3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e07cdf3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e07cdf3d
Branch: refs/heads/KYLIN-943
Commit: e07cdf3d0b4c36eccf2dabd333ec5a6c03af1d94
Parents: cc660bf
Author: shaofengshi <sh...@apache.org>
Authored: Mon Sep 21 17:10:07 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Sep 21 17:10:07 2015 +0800
----------------------------------------------------------------------
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 17 +-------
.../localmeta/table/DEFAULT.SREAMING_TABLE.json | 43 --------------------
.../table/DEFAULT.STREAMING_TABLE.json | 43 ++++++++++++++++++++
3 files changed, 44 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e07cdf3d/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index f3cf6f5..a10fef4 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -84,14 +83,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
segment.setInputRecordsSize(sourceSizeBytes);
try {
- if (isBuildAndMerge(cubingJob)) {
- // don't mark the segment ready yet, it's going to be merged right after
- CubeUpdate cubeBuilder = new CubeUpdate(cube);
- cubeBuilder.setToUpdateSegs(segment);
- cubeManager.updateCube(cubeBuilder);
- } else {
- cubeManager.promoteNewlyBuiltSegments(cube, segment);
- }
+ cubeManager.promoteNewlyBuiltSegments(cube, segment);
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
} catch (IOException e) {
logger.error("fail to update cube after build", e);
@@ -99,11 +91,4 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
}
}
- private boolean isBuildAndMerge(CubingJob cubingJob) {
- for (AbstractExecutable task : cubingJob.getTasks()) {
- if (task instanceof UpdateCubeInfoAfterMergeStep)
- return true;
- }
- return false;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e07cdf3d/examples/test_case_data/localmeta/table/DEFAULT.SREAMING_TABLE.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.SREAMING_TABLE.json b/examples/test_case_data/localmeta/table/DEFAULT.SREAMING_TABLE.json
deleted file mode 100644
index 3033d91..0000000
--- a/examples/test_case_data/localmeta/table/DEFAULT.SREAMING_TABLE.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- "uuid": "e286e39e-40d7-44c2-8fa2-41b365123987",
- "name": "STREAMING_TABLE",
- "columns": [
- {
- "id": "1",
- "name": "minute_start",
- "datatype": "timestamp"
- },
- {
- "id": "2",
- "name": "hour_start",
- "datatype": "timestamp"
- },
- {
- "id": "3",
- "name": "day_start",
- "datatype": "date"
- },
- {
- "id": "4",
- "name": "itm",
- "datatype": "string"
- },
- {
- "id": "5",
- "name": "site",
- "datatype": "string"
- },
- {
- "id": "6",
- "name": "gmv",
- "datatype": "decimal(19,6)"
- },
- {
- "id": "7",
- "name": "item_count",
- "datatype": "int"
- }
- ],
- "database": "DEFAULT",
- "last_modified": 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e07cdf3d/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json b/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
new file mode 100644
index 0000000..c9a1f98
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
@@ -0,0 +1,43 @@
+{
+ "uuid": "e286e39e-40d7-44c2-8fa2-41b365123987",
+ "name": "STREAMING_TABLE",
+ "columns": [
+ {
+ "id": "1",
+ "name": "minute_start",
+ "datatype": "timestamp"
+ },
+ {
+ "id": "2",
+ "name": "hour_start",
+ "datatype": "timestamp"
+ },
+ {
+ "id": "3",
+ "name": "day_start",
+ "datatype": "date"
+ },
+ {
+ "id": "4",
+ "name": "itm",
+ "datatype": "string"
+ },
+ {
+ "id": "5",
+ "name": "site",
+ "datatype": "string"
+ },
+ {
+ "id": "6",
+ "name": "gmv",
+ "datatype": "decimal(19,6)"
+ },
+ {
+ "id": "7",
+ "name": "item_count",
+ "datatype": "int"
+ }
+ ],
+ "database": "DEFAULT",
+ "last_modified": 0
+}
\ No newline at end of file