You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/04/08 11:55:48 UTC
[06/35] kylin git commit: minor,
move JobInfoConverter to kylin-engine-mr
minor, move JobInfoConverter to kylin-engine-mr
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/598b32ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/598b32ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/598b32ef
Branch: refs/heads/master-hbase0.98
Commit: 598b32efa21ebd87e76ac5798b30f472c6e29de2
Parents: 997d429
Author: Yifan Zhang <ev...@gmail.com>
Authored: Wed Mar 29 10:29:16 2017 +0800
Committer: Dong Li <li...@apache.org>
Committed: Wed Mar 29 10:47:10 2017 +0800
----------------------------------------------------------------------
.../engine/mr/common/JobInfoConverter.java | 128 ++++++++++++++++++
.../apache/kylin/rest/service/JobService.java | 2 +-
.../kylin/rest/util/JobInfoConverter.java | 130 -------------------
3 files changed, 129 insertions(+), 131 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/598b32ef/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
new file mode 100644
index 0000000..189e019
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import java.util.Map;
+
+import org.apache.kylin.cube.model.CubeBuildTypeEnum;
+import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
+import org.apache.kylin.job.JobInstance;
+import org.apache.kylin.job.common.ShellExecutable;
+import org.apache.kylin.job.constant.JobStatusEnum;
+import org.apache.kylin.job.constant.JobStepStatusEnum;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.execution.Output;
+
+import com.google.common.base.Preconditions;
+
+public class JobInfoConverter {
+ public static JobInstance parseToJobInstance(AbstractExecutable job, Map<String, Output> outputs) {
+ if (job == null) {
+ return null;
+ }
+ Preconditions.checkState(job instanceof CubingJob, "illegal job type, id:" + job.getId());
+ CubingJob cubeJob = (CubingJob) job;
+ Output output = outputs.get(job.getId());
+ final JobInstance result = new JobInstance();
+ result.setName(job.getName());
+ result.setRelatedCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
+ result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
+ result.setLastModified(output.getLastModified());
+ result.setSubmitter(cubeJob.getSubmitter());
+ result.setUuid(cubeJob.getId());
+ result.setType(CubeBuildTypeEnum.BUILD);
+ result.setStatus(parseToJobStatus(output.getState()));
+ result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
+ result.setExecStartTime(AbstractExecutable.getStartTime(output));
+ result.setExecEndTime(AbstractExecutable.getEndTime(output));
+ result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime()) / 1000);
+ for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
+ AbstractExecutable task = cubeJob.getTasks().get(i);
+ result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
+ }
+ return result;
+ }
+
+ public static JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i, Output stepOutput) {
+ Preconditions.checkNotNull(stepOutput);
+ JobInstance.JobStep result = new JobInstance.JobStep();
+ result.setId(task.getId());
+ result.setName(task.getName());
+ result.setSequenceID(i);
+ result.setStatus(parseToJobStepStatus(stepOutput.getState()));
+ for (Map.Entry<String, String> entry : stepOutput.getExtra().entrySet()) {
+ if (entry.getKey() != null && entry.getValue() != null) {
+ result.putInfo(entry.getKey(), entry.getValue());
+ }
+ }
+ result.setExecStartTime(AbstractExecutable.getStartTime(stepOutput));
+ result.setExecEndTime(AbstractExecutable.getEndTime(stepOutput));
+ if (task instanceof ShellExecutable) {
+ result.setExecCmd(((ShellExecutable) task).getCmd());
+ }
+ if (task instanceof MapReduceExecutable) {
+ result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams());
+ result.setExecWaitTime(AbstractExecutable.getExtraInfoAsLong(stepOutput, MapReduceExecutable.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
+ }
+ if (task instanceof HadoopShellExecutable) {
+ result.setExecCmd(((HadoopShellExecutable) task).getJobParams());
+ }
+ return result;
+ }
+
+ public static JobStatusEnum parseToJobStatus(ExecutableState state) {
+ switch (state) {
+ case READY:
+ return JobStatusEnum.PENDING;
+ case RUNNING:
+ return JobStatusEnum.RUNNING;
+ case ERROR:
+ return JobStatusEnum.ERROR;
+ case DISCARDED:
+ return JobStatusEnum.DISCARDED;
+ case SUCCEED:
+ return JobStatusEnum.FINISHED;
+ case STOPPED:
+ return JobStatusEnum.STOPPED;
+ default:
+ throw new RuntimeException("invalid state:" + state);
+ }
+ }
+
+ public static JobStepStatusEnum parseToJobStepStatus(ExecutableState state) {
+ switch (state) {
+ case READY:
+ return JobStepStatusEnum.PENDING;
+ case RUNNING:
+ return JobStepStatusEnum.RUNNING;
+ case ERROR:
+ return JobStepStatusEnum.ERROR;
+ case DISCARDED:
+ return JobStepStatusEnum.DISCARDED;
+ case SUCCEED:
+ return JobStepStatusEnum.FINISHED;
+ case STOPPED:
+ return JobStepStatusEnum.STOPPED;
+ default:
+ throw new RuntimeException("invalid state:" + state);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/598b32ef/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 1e91b43..4ba426e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -37,6 +37,7 @@ import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.engine.mr.common.JobInfoConverter;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.Scheduler;
@@ -55,7 +56,6 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.BadRequestException;
-import org.apache.kylin.rest.util.JobInfoConverter;
import org.apache.kylin.source.ISource;
import org.apache.kylin.source.SourceFactory;
import org.apache.kylin.source.SourcePartition;
http://git-wip-us.apache.org/repos/asf/kylin/blob/598b32ef/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java b/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java
deleted file mode 100644
index e13c9c3..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.util;
-
-import java.util.Map;
-
-import org.apache.kylin.cube.model.CubeBuildTypeEnum;
-import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
-import org.apache.kylin.engine.mr.common.MapReduceExecutable;
-import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
-import org.apache.kylin.job.JobInstance;
-import org.apache.kylin.job.common.ShellExecutable;
-import org.apache.kylin.job.constant.JobStatusEnum;
-import org.apache.kylin.job.constant.JobStepStatusEnum;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.execution.Output;
-
-import com.google.common.base.Preconditions;
-
-public class JobInfoConverter {
- public static JobInstance parseToJobInstance(AbstractExecutable job, Map<String, Output> outputs) {
- if (job == null) {
- return null;
- }
- Preconditions.checkState(job instanceof CubingJob, "illegal job type, id:" + job.getId());
- CubingJob cubeJob = (CubingJob) job;
- Output output = outputs.get(job.getId());
- final JobInstance result = new JobInstance();
- result.setName(job.getName());
- result.setRelatedCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
- result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
- result.setLastModified(output.getLastModified());
- result.setSubmitter(cubeJob.getSubmitter());
- result.setUuid(cubeJob.getId());
- result.setType(CubeBuildTypeEnum.BUILD);
- result.setStatus(parseToJobStatus(output.getState()));
- result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
- result.setExecStartTime(AbstractExecutable.getStartTime(output));
- result.setExecEndTime(AbstractExecutable.getEndTime(output));
- result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime()) / 1000);
- for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
- AbstractExecutable task = cubeJob.getTasks().get(i);
- result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
- }
- return result;
- }
-
- public static JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i, Output stepOutput) {
- Preconditions.checkNotNull(stepOutput);
- JobInstance.JobStep result = new JobInstance.JobStep();
- result.setId(task.getId());
- result.setName(task.getName());
- result.setSequenceID(i);
- result.setStatus(parseToJobStepStatus(stepOutput.getState()));
- for (Map.Entry<String, String> entry : stepOutput.getExtra().entrySet()) {
- if (entry.getKey() != null && entry.getValue() != null) {
- result.putInfo(entry.getKey(), entry.getValue());
- }
- }
- result.setExecStartTime(AbstractExecutable.getStartTime(stepOutput));
- result.setExecEndTime(AbstractExecutable.getEndTime(stepOutput));
- if (task instanceof ShellExecutable) {
- result.setExecCmd(((ShellExecutable) task).getCmd());
- }
- if (task instanceof MapReduceExecutable) {
- result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams());
- result.setExecWaitTime(AbstractExecutable.getExtraInfoAsLong(stepOutput, MapReduceExecutable.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
- }
- if (task instanceof HadoopShellExecutable) {
- result.setExecCmd(((HadoopShellExecutable) task).getJobParams());
- }
- return result;
- }
-
- public static JobStatusEnum parseToJobStatus(ExecutableState state) {
- switch (state) {
- case READY:
- return JobStatusEnum.PENDING;
- case RUNNING:
- return JobStatusEnum.RUNNING;
- case ERROR:
- return JobStatusEnum.ERROR;
- case DISCARDED:
- return JobStatusEnum.DISCARDED;
- case SUCCEED:
- return JobStatusEnum.FINISHED;
- case STOPPED:
- return JobStatusEnum.STOPPED;
- default:
- throw new RuntimeException("invalid state:" + state);
- }
- }
-
- public static JobStepStatusEnum parseToJobStepStatus(ExecutableState state) {
- switch (state) {
- case READY:
- return JobStepStatusEnum.PENDING;
- case RUNNING:
- return JobStepStatusEnum.RUNNING;
- case ERROR:
- return JobStepStatusEnum.ERROR;
- case DISCARDED:
- return JobStepStatusEnum.DISCARDED;
- case SUCCEED:
- return JobStepStatusEnum.FINISHED;
- case STOPPED:
- return JobStepStatusEnum.STOPPED;
- default:
- throw new RuntimeException("invalid state:" + state);
- }
- }
-}