You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by nj...@apache.org on 2016/04/18 11:45:42 UTC
kylin git commit: Add Env and Project Info in job email notification
Repository: kylin
Updated Branches:
refs/heads/1.4-rc 08b2051f8 -> 85113c11f
Add Env and Project Info in job email notification
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85113c11
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85113c11
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85113c11
Branch: refs/heads/1.4-rc
Commit: 85113c11f443c8ee702c5a8d201b926349e80306
Parents: 08b2051
Author: Zhong <ya...@lm-shc-16501214.corp.ebay.com>
Authored: Mon Apr 18 17:39:57 2016 +0800
Committer: Zhong <ya...@lm-shc-16501214.corp.ebay.com>
Committed: Mon Apr 18 17:39:57 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 11 ++--
.../kylin/job/constant/ExecutableConstants.java | 2 +-
.../org/apache/kylin/engine/mr/CubingJob.java | 56 ++++++++++++++++----
3 files changed, 52 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/85113c11/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 9bbcbb4..6f1fccc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -173,10 +173,7 @@ public class KylinConfigBase implements Serializable {
if (!root.endsWith("/")) {
root += "/";
}
- return new StringBuffer(root)
- .append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-'))
- .append("/")
- .toString();
+ return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString();
}
public CliCommandExecutor getCliCommandExecutor() throws IOException {
@@ -485,7 +482,7 @@ public class KylinConfigBase implements Serializable {
public boolean isGetJobStatusWithKerberos() {
return Boolean.valueOf(this.getOptional("kylin.job.status.with.kerberos", "false"));
}
-
+
public boolean isTransformPathToMasterNN() {
return Boolean.valueOf(this.getOptional("kylin.transform.hdfs.path.enable", "false"));
}
@@ -526,6 +523,10 @@ public class KylinConfigBase implements Serializable {
return getOptional("mail.sender", "");
}
+ public String getDeployEnv() {
+ return getOptional("deploy.env", "DEV");
+ }
+
public String toString() {
return getMetadataUrl();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/85113c11/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index d370b0d..ab2fed7 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -75,5 +75,5 @@ public final class ExecutableConstants {
public static final String PROP_JOB_KILLED = "jobKilled";
public static final String PROP_JOB_RUNTIME_FLOWS = "jobFlows";
- public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" + "<li>Cube Name: ${cube_name}</li>" + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>";
+ public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" + "<li>Env: ${env_name}</li>" + "<li>Project: ${project_name}</li>" + "<li>Cube Name: ${cube_name}</li>" + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>";
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/85113c11/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 979ff75..1353fe8 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -30,6 +30,7 @@ import java.util.regex.Matcher;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -43,11 +44,13 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.job.execution.Output;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
/**
*/
public class CubingJob extends DefaultChainedExecutable {
-
+
public static enum AlgorithmEnum {
LAYER, INMEM
}
@@ -58,6 +61,8 @@ public class CubingJob extends DefaultChainedExecutable {
public static final String CUBE_SIZE_BYTES = "byteSizeBytes";
public static final String MAP_REDUCE_WAIT_TIME = "mapReduceWaitTime";
+ private static final String DEPLOY_ENV_NAME = "envName";
+ private static final String PROJECT_INSTANCE_NAME = "projectName";
private static final String CUBE_INSTANCE_NAME = "cubeName";
private static final String SEGMENT_ID = "segmentId";
@@ -70,10 +75,21 @@ public class CubingJob extends DefaultChainedExecutable {
}
private static CubingJob initCubingJob(CubeSegment seg, String jobType, String submitter, JobEngineConfig config) {
+ KylinConfig kylinConfig = config.getConfig();
+ CubeInstance cube = seg.getCubeInstance();
+ List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(),cube.getName());
+ if(projList==null || projList.size()==0){
+ throw new RuntimeException("Cannot find the project containing the cube " +cube.getName()+"!!!");
+ }else if(projList.size()>=2){
+ throw new RuntimeException("Find more than one project containing the cube "+cube.getName()+". It does't meet the uniqueness requirement!!! ");
+ }
+
CubingJob result = new CubingJob();
SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss");
format.setTimeZone(TimeZone.getTimeZone(config.getTimeZone()));
- result.setCubeName(seg.getCubeInstance().getName());
+ result.setDeployEnvName(kylinConfig.getDeployEnv());
+ result.setProjectName(projList.get(0).getName());
+ result.setCubeName(cube.getName());
result.setSegmentId(seg.getUuid());
result.setName(seg.getCubeInstance().getName() + " - " + seg.getName() + " - " + jobType + " - " + format.format(new Date(System.currentTimeMillis())));
result.setSubmitter(submitter);
@@ -85,6 +101,22 @@ public class CubingJob extends DefaultChainedExecutable {
super();
}
+ void setDeployEnvName(String name) {
+ setParam(DEPLOY_ENV_NAME, name);
+ }
+
+ public String getDeployEnvName() {
+ return getParam(DEPLOY_ENV_NAME);
+ }
+
+ void setProjectName(String name) {
+ setParam(PROJECT_INSTANCE_NAME, name);
+ }
+
+ public String getProjectName() {
+ return getParam(PROJECT_INSTANCE_NAME);
+ }
+
void setCubeName(String name) {
setParam(CUBE_INSTANCE_NAME, name);
}
@@ -131,6 +163,8 @@ public class CubingJob extends DefaultChainedExecutable {
String content = ExecutableConstants.NOTIFY_EMAIL_TEMPLATE;
content = content.replaceAll("\\$\\{job_name\\}", getName());
content = content.replaceAll("\\$\\{result\\}", state.toString());
+ content = content.replaceAll("\\$\\{env_name\\}", getDeployEnvName());
+ content = content.replaceAll("\\$\\{project_name\\}", getProjectName());
content = content.replaceAll("\\$\\{cube_name\\}", getCubeName());
content = content.replaceAll("\\$\\{source_records_count\\}", String.valueOf(findSourceRecordCount()));
content = content.replaceAll("\\$\\{start_time\\}", new Date(getStartTime()).toString());
@@ -147,7 +181,7 @@ public class CubingJob extends DefaultChainedExecutable {
logger.warn(e.getLocalizedMessage(), e);
}
- String title = "[" + state.toString() + "] - [Kylin Cube Build Job]-" + getCubeName();
+ String title = "[" + state.toString() + "] - [" + getDeployEnvName() + "] - [" + getProjectName() + "] - " + getCubeName();
return Pair.of(title, content);
}
@@ -174,11 +208,11 @@ public class CubingJob extends DefaultChainedExecutable {
public void setMapReduceWaitTime(long t) {
addExtraInfo(MAP_REDUCE_WAIT_TIME, t + "");
}
-
+
public void setAlgorithm(AlgorithmEnum alg) {
addExtraInfo("algorithm", alg.name());
}
-
+
public AlgorithmEnum getAlgorithm() {
String alg = getExtraInfo().get("algorithm");
return alg == null ? null : AlgorithmEnum.valueOf(alg);
@@ -187,11 +221,11 @@ public class CubingJob extends DefaultChainedExecutable {
public boolean isLayerCubing() {
return AlgorithmEnum.LAYER == getAlgorithm();
}
-
+
public boolean isInMemCubing() {
return AlgorithmEnum.INMEM == getAlgorithm();
}
-
+
public long findSourceRecordCount() {
return Long.parseLong(findExtraInfo(SOURCE_RECORD_COUNT, "0"));
}
@@ -204,7 +238,7 @@ public class CubingJob extends DefaultChainedExecutable {
// look for the info BACKWARD, let the last step that claims the cube size win
return Long.parseLong(findExtraInfoBackward(CUBE_SIZE_BYTES, "0"));
}
-
+
public String findExtraInfo(String key, String dft) {
return findExtraInfo(key, dft, false);
}
@@ -212,14 +246,14 @@ public class CubingJob extends DefaultChainedExecutable {
public String findExtraInfoBackward(String key, String dft) {
return findExtraInfo(key, dft, true);
}
-
+
private String findExtraInfo(String key, String dft, boolean backward) {
ArrayList<AbstractExecutable> tasks = new ArrayList<AbstractExecutable>(getTasks());
-
+
if (backward) {
Collections.reverse(tasks);
}
-
+
for (AbstractExecutable child : tasks) {
Output output = executableManager.getOutput(child.getId());
String value = output.getExtra().get(key);