You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/03/12 11:49:10 UTC
[38/48] kylin git commit: KYLIN-2503 Spark cubing step should show
YARN app link
KYLIN-2503 Spark cubing step should show YARN app link
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f38d871a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f38d871a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f38d871a
Branch: refs/heads/master-hbase0.98
Commit: f38d871a72e1e24331e092fd51664fcb8fd47e22
Parents: 5d012b9
Author: shaofengshi <sh...@apache.org>
Authored: Sat Mar 11 22:08:28 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Mar 11 22:08:28 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/job/common/ShellExecutable.java | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f38d871a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
index a68f242..5e0d0db 100644
--- a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
@@ -83,6 +83,10 @@ public class ShellExecutable extends AbstractExecutable {
private static final Pattern PATTERN_HIVE_APP_ID_URL = Pattern.compile("Starting Job = (.*?), Tracking URL = (.*)");
private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write: (\\d+) SUCCESS");
+ // spark
+ private static final Pattern PATTERN_SPARK_APP_ID = Pattern.compile("Submitted application (.*?)");
+ private static final Pattern PATTERN_SPARK_APP_URL = Pattern.compile("tracking URL: (.*)");
+
@Override
public void log(String message) {
Matcher matcher = PATTERN_APP_ID.matcher(message);
@@ -136,6 +140,19 @@ public class ShellExecutable extends AbstractExecutable {
String hdfsWritten = matcher.group(2);
info.put(ExecutableConstants.HDFS_BYTES_WRITTEN, hdfsWritten);
}
+
+ // spark
+ matcher = PATTERN_SPARK_APP_ID.matcher(message);
+ if (matcher.find()) {
+ String app_id = matcher.group(1);
+ info.put(ExecutableConstants.YARN_APP_ID, app_id);
+ }
+
+ matcher = PATTERN_SPARK_APP_URL.matcher(message);
+ if (matcher.find()) {
+ String trackingUrl = matcher.group(1);
+ info.put(ExecutableConstants.YARN_APP_URL, trackingUrl);
+ }
}
Map<String, String> getInfo() {