You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/05/17 13:35:42 UTC
[2/6] kylin git commit: KYLIN-2562 Allow configuring yarn app
tracking URL pattern
KYLIN-2562 Allow configuring yarn app tracking URL pattern
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/468eb3fa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/468eb3fa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/468eb3fa
Branch: refs/heads/2.0.x-hbase0.98
Commit: 468eb3fa2e8ed0ec8452d8bac3a747f03b1ea6a9
Parents: 044b346
Author: shaofengshi <sh...@apache.org>
Authored: Mon Apr 24 16:54:53 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed May 17 13:29:21 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 3 +++
.../kylin/job/common/PatternedLogger.java | 8 ++++++
.../kylin/job/execution/ExecutableManager.java | 26 ++++++++++++++++++++
3 files changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/468eb3fa/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index e314941..09bbb72 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -978,4 +978,7 @@ abstract public class KylinConfigBase implements Serializable {
return getResourceStoreImpls().get(key);
}
+ public String getJobTrackingURLPattern() {
+ return getOptional("kylin.job.tracking-url-pattern", "");
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/468eb3fa/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java b/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java
index 8399b44..8be5d02 100644
--- a/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java
+++ b/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java
@@ -45,6 +45,8 @@ public class PatternedLogger extends BufferedLogger {
private static final Pattern PATTERN_HIVE_APP_ID_URL = Pattern.compile("Starting Job = (.*?), Tracking URL = (.*)");
private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write: (\\d+) SUCCESS");
+ private static final Pattern PATTERN_HIVE_APP_ID_URL_2 = Pattern.compile("Executing on YARN cluster with App id (.*?)");
+
// spark
private static final Pattern PATTERN_SPARK_APP_ID = Pattern.compile("Submitted application (.*?)");
private static final Pattern PATTERN_SPARK_APP_URL = Pattern.compile("tracking URL: (.*)");
@@ -100,6 +102,12 @@ public class PatternedLogger extends BufferedLogger {
String trackingUrl = matcher.group(2);
info.put(ExecutableConstants.MR_JOB_ID, jobId);
info.put(ExecutableConstants.YARN_APP_URL, trackingUrl);
+ } else {
+ matcher = PATTERN_HIVE_APP_ID_URL_2.matcher(message);
+ if (matcher.find()) {
+ String jobId = matcher.group(1);
+ info.put(ExecutableConstants.YARN_APP_ID, jobId);
+ }
}
matcher = PATTERN_HIVE_BYTES_WRITTEN.matcher(message);
http://git-wip-us.apache.org/repos/asf/kylin/blob/468eb3fa/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 0c86d72..2272582 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -20,13 +20,16 @@ package org.apache.kylin.job.execution;
import java.lang.reflect.Constructor;
import java.util.HashMap;
+import java.util.IllegalFormatException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.dao.ExecutableDao;
import org.apache.kylin.job.dao.ExecutableOutputPO;
import org.apache.kylin.job.dao.ExecutablePO;
@@ -39,6 +42,10 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import static org.apache.kylin.job.constant.ExecutableConstants.MR_JOB_ID;
+import static org.apache.kylin.job.constant.ExecutableConstants.YARN_APP_ID;
+import static org.apache.kylin.job.constant.ExecutableConstants.YARN_APP_URL;
+
/**
*/
public class ExecutableManager {
@@ -412,6 +419,25 @@ public class ExecutableManager {
if (info == null) {
return;
}
+
+ // post process
+ if (info.containsKey(MR_JOB_ID) && !info.containsKey(ExecutableConstants.YARN_APP_ID)) {
+ String jobId = info.get(MR_JOB_ID);
+ if (jobId.startsWith("job_")) {
+ info.put(YARN_APP_ID, jobId.replace("job_", "application_"));
+ }
+ }
+
+ if (info.containsKey(YARN_APP_ID) && !StringUtils.isEmpty(config.getJobTrackingURLPattern())) {
+ String pattern = config.getJobTrackingURLPattern();
+ try {
+ String newTrackingURL = String.format(pattern, info.get(YARN_APP_ID));
+ info.put(YARN_APP_URL, newTrackingURL);
+ } catch (IllegalFormatException ife) {
+ logger.error("Illegal tracking url pattern: " + config.getJobTrackingURLPattern());
+ }
+ }
+
try {
ExecutableOutputPO output = executableDao.getJobOutput(id);
Preconditions.checkArgument(output != null, "there is no related output for job id:" + id);