You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/07/16 05:27:51 UTC
[11/11] incubator-kylin git commit: KYLIN-877 Add IMRInput javadoc
KYLIN-877 Add IMRInput javadoc
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b033d3aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b033d3aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b033d3aa
Branch: refs/heads/KYLIN-876
Commit: b033d3aac2f22dd431ab815ed967ffb422ce53ed
Parents: d109e27
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Jul 16 11:27:13 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jul 16 11:27:13 2015 +0800
----------------------------------------------------------------------
.../kylin/engine/mr/BatchCubingJobBuilder.java | 2 ++
.../org/apache/kylin/engine/mr/IMRInput.java | 22 ++++++++++++++++++++
2 files changed, 24 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b033d3aa/job/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java b/job/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
index c1524e7..239ce64 100644
--- a/job/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
+++ b/job/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
@@ -49,9 +49,11 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
// Phase 1: Create Flat Table
inputSide.addStepPhase1_CreateFlatTable(result);
+ // Phase 2: Build Dictionary
result.addTask(createFactDistinctColumnsStep(flatHiveTableDesc, jobId));
result.addTask(createBuildDictionaryStep(jobId));
+ // Phase 3: Build Cube
if (config.isInMemCubing()) {
result.addTask(createSaveStatisticsStep(jobId));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b033d3aa/job/src/main/java/org/apache/kylin/engine/mr/IMRInput.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/engine/mr/IMRInput.java b/job/src/main/java/org/apache/kylin/engine/mr/IMRInput.java
index c170b47..08ed94a 100644
--- a/job/src/main/java/org/apache/kylin/engine/mr/IMRInput.java
+++ b/job/src/main/java/org/apache/kylin/engine/mr/IMRInput.java
@@ -23,25 +23,47 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.metadata.model.TableDesc;
+/**
+ * Any ITableSource that wishes to serve as input of MapReduce build engine must adapt to this interface.
+ */
public interface IMRInput {
+ /** Return a helper to participate in batch cubing job flow. */
public IMRBatchCubingInputSide getBatchCubingInputSide(CubeSegment seg);
+ /** Return an InputFormat that reads from specified table. */
public IMRTableInputFormat getTableInputFormat(TableDesc table);
+ /**
+ * Utility that configures mapper to read from a table.
+ */
public interface IMRTableInputFormat {
+ /** Configure the InputFormat of given job. */
public void configureJob(Job job);
+ /** Parse a mapper input object into column values. */
public String[] parseMapperInput(Object mapperInput);
}
+ /**
+ * Participate the batch cubing flow as the input side. Responsible for creating
+ * intermediate flat table (Phase 1) and clean up if necessary (Phase 4).
+ *
+ * - Phase 1: Create Flat Table
+ * - Phase 2: Build Dictionary
+ * - Phase 3: Build Cube
+ * - Phase 4: Update Metadata & Cleanup
+ */
public interface IMRBatchCubingInputSide {
+ /** Add step that creates an intermediate flat table as defined by CubeJoinedFlatTableDesc */
public void addStepPhase1_CreateFlatTable(DefaultChainedExecutable jobFlow);
+ /** Add step that does necessary clean up, like delete the intermediate flat table */
public void addStepPhase4_UpdateMetadataAndCleanup(DefaultChainedExecutable jobFlow);
+ /** Return an InputFormat that reads from the intermediate flat table */
public IMRTableInputFormat getFlatTableInputFormat();
}
}