You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/07/10 11:05:18 UTC
incubator-kylin git commit: KYLIN-876 separate cubing and streaming
engine
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-876 [created] bbd5e27e0
KYLIN-876 separate cubing and streaming engine
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bbd5e27e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bbd5e27e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bbd5e27e
Branch: refs/heads/KYLIN-876
Commit: bbd5e27e04b723346727e49ae9871b15b2102fbb
Parents: 3011f6f
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Jul 10 16:59:48 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Jul 10 16:59:48 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/BuildEngineFactory.java | 18 +++-----
.../apache/kylin/engine/IBatchCubingEngine.java | 32 +++++++++++++++
.../org/apache/kylin/engine/IBuildEngine.java | 34 ----------------
.../kylin/engine/IStreamingCubingEngine.java | 8 ++++
.../kylin/engine/mr/MRBatchCubingEngine.java | 37 +++++++++++++++++
.../apache/kylin/engine/mr/MRBuildEngine.java | 43 --------------------
.../kylin/job/BuildCubeWithEngineTest.java | 2 +-
.../apache/kylin/rest/service/JobService.java | 4 +-
8 files changed, 85 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bbd5e27e/job/src/main/java/org/apache/kylin/engine/BuildEngineFactory.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/engine/BuildEngineFactory.java b/job/src/main/java/org/apache/kylin/engine/BuildEngineFactory.java
index 7f3c761..0e31f6b 100644
--- a/job/src/main/java/org/apache/kylin/engine/BuildEngineFactory.java
+++ b/job/src/main/java/org/apache/kylin/engine/BuildEngineFactory.java
@@ -19,29 +19,21 @@
package org.apache.kylin.engine;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.engine.mr.MRBuildEngine;
+import org.apache.kylin.engine.mr.MRBatchCubingEngine;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
public class BuildEngineFactory {
- private static final IBuildEngine defaultEngine = new MRBuildEngine();
-
- public static IBuildEngine getDefaultEngine() {
- return defaultEngine;
- }
-
- public static Runnable createStreamingCubeBuilder(CubeSegment newSegment, String submitter) {
- return getDefaultEngine().createStreamingCubeBuilder(newSegment, submitter);
- }
+ private static final IBatchCubingEngine defaultBatch = new MRBatchCubingEngine();
/** Build a new cube segment, typically its time range appends to the end of current cube. */
- public static DefaultChainedExecutable createBatchBuildJob(CubeSegment newSegment, String submitter) {
- return getDefaultEngine().createBatchBuildJob(newSegment, submitter);
+ public static DefaultChainedExecutable createBatchCubingJob(CubeSegment newSegment, String submitter) {
+ return defaultBatch.createBatchCubingJob(newSegment, submitter);
}
/** Merge multiple small segments into a big one. */
public static DefaultChainedExecutable createBatchMergeJob(CubeSegment mergeSegment, String submitter) {
- return getDefaultEngine().createBatchMergeJob(mergeSegment, submitter);
+ return defaultBatch.createBatchMergeJob(mergeSegment, submitter);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bbd5e27e/job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java b/job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java
new file mode 100644
index 0000000..55a080c
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine;
+
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.job.execution.DefaultChainedExecutable;
+
+public interface IBatchCubingEngine {
+
+ /** Build a new cube segment, typically its time range appends to the end of current cube. */
+ public DefaultChainedExecutable createBatchCubingJob(CubeSegment newSegment, String submitter);
+
+ /** Merge multiple small segments into a big one. */
+ public DefaultChainedExecutable createBatchMergeJob(CubeSegment mergeSegment, String submitter);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bbd5e27e/job/src/main/java/org/apache/kylin/engine/IBuildEngine.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/engine/IBuildEngine.java b/job/src/main/java/org/apache/kylin/engine/IBuildEngine.java
deleted file mode 100644
index 909004c..0000000
--- a/job/src/main/java/org/apache/kylin/engine/IBuildEngine.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine;
-
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.job.execution.DefaultChainedExecutable;
-
-public interface IBuildEngine {
-
- public Runnable createStreamingCubeBuilder(CubeSegment newSegment, String submitter);
-
- /** Build a new cube segment, typically its time range appends to the end of current cube. */
- public DefaultChainedExecutable createBatchBuildJob(CubeSegment newSegment, String submitter);
-
- /** Merge multiple small segments into a big one. */
- public DefaultChainedExecutable createBatchMergeJob(CubeSegment mergeSegment, String submitter);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bbd5e27e/job/src/main/java/org/apache/kylin/engine/IStreamingCubingEngine.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/engine/IStreamingCubingEngine.java b/job/src/main/java/org/apache/kylin/engine/IStreamingCubingEngine.java
new file mode 100644
index 0000000..0359ce9
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/engine/IStreamingCubingEngine.java
@@ -0,0 +1,8 @@
+package org.apache.kylin.engine;
+
+import org.apache.kylin.cube.CubeSegment;
+
+public interface IStreamingCubingEngine {
+
+ public Runnable createStreamingCubingBuilder(CubeSegment seg);
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bbd5e27e/job/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine.java b/job/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine.java
new file mode 100644
index 0000000..332f20e
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr;
+
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.IBatchCubingEngine;
+import org.apache.kylin.job.execution.DefaultChainedExecutable;
+
+public class MRBatchCubingEngine implements IBatchCubingEngine {
+
+ @Override
+ public DefaultChainedExecutable createBatchCubingJob(CubeSegment newSegment, String submitter) {
+ return new BatchCubingJobBuilder(newSegment, submitter).build();
+ }
+
+ @Override
+ public DefaultChainedExecutable createBatchMergeJob(CubeSegment mergeSegment, String submitter) {
+ return new BatchMergeJobBuilder(mergeSegment, submitter).build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bbd5e27e/job/src/main/java/org/apache/kylin/engine/mr/MRBuildEngine.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/engine/mr/MRBuildEngine.java b/job/src/main/java/org/apache/kylin/engine/mr/MRBuildEngine.java
deleted file mode 100644
index 3c482b4..0000000
--- a/job/src/main/java/org/apache/kylin/engine/mr/MRBuildEngine.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.mr;
-
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.engine.IBuildEngine;
-import org.apache.kylin.job.execution.DefaultChainedExecutable;
-
-public class MRBuildEngine implements IBuildEngine {
-
- @Override
- public Runnable createStreamingCubeBuilder(CubeSegment newSegment, String submitter) {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public DefaultChainedExecutable createBatchBuildJob(CubeSegment newSegment, String submitter) {
- return new BatchCubingJobBuilder(newSegment, submitter).build();
- }
-
- @Override
- public DefaultChainedExecutable createBatchMergeJob(CubeSegment mergeSegment, String submitter) {
- return new BatchMergeJobBuilder(mergeSegment, submitter).build();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bbd5e27e/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 066cdd6..4db1cde 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -275,7 +275,7 @@ public class BuildCubeWithEngineTest {
private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
- DefaultChainedExecutable job = BuildEngineFactory.createBatchBuildJob(segment, "TEST");
+ DefaultChainedExecutable job = BuildEngineFactory.createBatchCubingJob(segment, "TEST");
jobService.addJob(job);
waitForJob(job.getId());
return job.getId();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bbd5e27e/server/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
index c799cb7..83e7604 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -136,13 +136,13 @@ public class JobService extends BasicService {
if (buildType == CubeBuildTypeEnum.BUILD) {
CubeSegment newSeg = getCubeManager().appendSegments(cube, endDate);
- job = BuildEngineFactory.createBatchBuildJob(newSeg, submitter);
+ job = BuildEngineFactory.createBatchCubingJob(newSeg, submitter);
} else if (buildType == CubeBuildTypeEnum.MERGE) {
CubeSegment newSeg = getCubeManager().mergeSegments(cube, startDate, endDate, forceMergeEmptySeg);
job = BuildEngineFactory.createBatchMergeJob(newSeg, submitter);
} else if (buildType == CubeBuildTypeEnum.REFRESH) {
CubeSegment refreshSeg = getCubeManager().refreshSegment(cube, startDate, endDate);
- job = BuildEngineFactory.createBatchBuildJob(refreshSeg, submitter);
+ job = BuildEngineFactory.createBatchCubingJob(refreshSeg, submitter);
} else {
throw new JobException("invalid build type:" + buildType);
}