You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by zh...@apache.org on 2016/03/14 09:47:33 UTC
kylin git commit: KYLIN-672 add env and project info in job email
notification patch from @Zhong Yanghong
Repository: kylin
Updated Branches:
refs/heads/master 75027ed64 -> 3800e4eb1
KYLIN-672 add env and project info in job email notification patch from @Zhong Yanghong
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3800e4eb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3800e4eb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3800e4eb
Branch: refs/heads/master
Commit: 3800e4eb114a64784e15932fbccabbd5423ac0f7
Parents: 75027ed
Author: Jason <ji...@163.com>
Authored: Mon Mar 14 16:46:51 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Mon Mar 14 16:47:23 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 ++
.../kylin/job/constant/ExecutableConstants.java | 3 +-
.../org/apache/kylin/engine/mr/CubingJob.java | 65 +++++++++++++++-----
3 files changed, 54 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3800e4eb/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index e3a73e8..4ee1947 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -579,4 +579,8 @@ public class KylinConfigBase implements Serializable {
return getOptional("kylin.hive.beeline.params", "");
}
+ public String getDeployEnv() {
+ return getOptional("deploy.env", "DEV");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3800e4eb/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index d3c1003..f619a68 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -54,6 +54,5 @@ public final class ExecutableConstants {
public static final String STEP_NAME_BUILD_II = "Build Inverted Index";
public static final String STEP_NAME_CONVERT_II_TO_HFILE = "Convert Inverted Index Data to HFile";
public static final String STEP_NAME_UPDATE_II_INFO = "Update Inverted Index Info";
-
- public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" + "<li>Cube Name: ${cube_name}</li>" + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>";
+ public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" + "<li>Env: ${env_name}</li>" + "<li>Project: ${project_name}</li>" + "<li>Cube Name: ${cube_name}</li>" + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>";
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3800e4eb/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 0325a09..7428a2f 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -21,13 +21,11 @@ package org.apache.kylin.engine.mr;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.TimeZone;
+import java.util.*;
import java.util.regex.Matcher;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -42,6 +40,8 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.job.execution.Output;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
/**
*/
@@ -56,6 +56,8 @@ public class CubingJob extends DefaultChainedExecutable {
public static final String SOURCE_SIZE_BYTES = "sourceSizeBytes";
public static final String CUBE_SIZE_BYTES = "byteSizeBytes";
public static final String MAP_REDUCE_WAIT_TIME = "mapReduceWaitTime";
+ private static final String DEPLOY_ENV_NAME = "envName";
+ private static final String PROJECT_INSTANCE_NAME = "projectName";
public static CubingJob createBuildJob(CubeSegment seg, String submitter, JobEngineConfig config) {
return initCubingJob(seg, "BUILD", submitter, config);
@@ -66,9 +68,20 @@ public class CubingJob extends DefaultChainedExecutable {
}
private static CubingJob initCubingJob(CubeSegment seg, String jobType, String submitter, JobEngineConfig config) {
+ KylinConfig kylinConfig = config.getConfig();
+ CubeInstance cube = seg.getCubeInstance();
+ List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(), cube.getName());
+ if (projList == null || projList.size() == 0) {
+ throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!");
+ } else if (projList.size() >= 2) {
+ throw new RuntimeException("Find more than one project containing the cube " + cube.getName() + ". It does't meet the uniqueness requirement!!! ");
+ }
+
CubingJob result = new CubingJob();
SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss");
format.setTimeZone(TimeZone.getTimeZone(config.getTimeZone()));
+ result.setDeployEnvName(kylinConfig.getDeployEnv());
+ result.setProjectName(projList.get(0).getName());
CubingExecutableUtil.setCubeName(seg.getCubeInstance().getName(), result.getParams());
CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
result.setName(seg.getCubeInstance().getName() + " - " + seg.getName() + " - " + jobType + " - " + format.format(new Date(System.currentTimeMillis())));
@@ -81,6 +94,23 @@ public class CubingJob extends DefaultChainedExecutable {
super();
}
+
+ void setDeployEnvName(String name) {
+ setParam(DEPLOY_ENV_NAME, name);
+ }
+
+ public static String getDeployEnvName() {
+ return DEPLOY_ENV_NAME;
+ }
+
+ void setProjectName(String name) {
+ setParam(PROJECT_INSTANCE_NAME, name);
+ }
+
+ public static String getProjectName() {
+ return PROJECT_INSTANCE_NAME;
+ }
+
@Override
protected Pair<String, String> formatNotifications(ExecutableContext context, ExecutableState state) {
CubeInstance cubeInstance = CubeManager.getInstance(context.getConfig()).getCube(CubingExecutableUtil.getCubeName(this.getParams()));
@@ -92,21 +122,23 @@ public class CubingJob extends DefaultChainedExecutable {
return null;
}
switch (state) {
- case ERROR:
- logMsg = output.getVerboseMsg();
- break;
- case DISCARDED:
- logMsg = "job has been discarded";
- break;
- case SUCCEED:
- logMsg = "job has succeeded";
- break;
- default:
- return null;
+ case ERROR:
+ logMsg = output.getVerboseMsg();
+ break;
+ case DISCARDED:
+ logMsg = "job has been discarded";
+ break;
+ case SUCCEED:
+ logMsg = "job has succeeded";
+ break;
+ default:
+ return null;
}
String content = ExecutableConstants.NOTIFY_EMAIL_TEMPLATE;
content = content.replaceAll("\\$\\{job_name\\}", getName());
content = content.replaceAll("\\$\\{result\\}", state.toString());
+ content = content.replaceAll("\\$\\{env_name\\}", getDeployEnvName());
+ content = content.replaceAll("\\$\\{project_name\\}", getProjectName());
content = content.replaceAll("\\$\\{cube_name\\}", CubingExecutableUtil.getCubeName(this.getParams()));
content = content.replaceAll("\\$\\{source_records_count\\}", String.valueOf(findSourceRecordCount()));
content = content.replaceAll("\\$\\{start_time\\}", new Date(getStartTime()).toString());
@@ -123,7 +155,8 @@ public class CubingJob extends DefaultChainedExecutable {
logger.warn(e.getLocalizedMessage(), e);
}
- String title = "[" + state.toString() + "] - [Kylin Cube Build Job]-" + CubingExecutableUtil.getCubeName(this.getParams());
+ String title = "[" + state.toString() + "] - [" + getDeployEnvName() + "] - [" + getProjectName() + "] - " + CubingExecutableUtil.getCubeName(this.getParams());
+
return Pair.of(title, content);
}