You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by qi...@apache.org on 2016/11/17 05:19:00 UTC
incubator-eagle git commit: [MINOR] add jobConf.additional.info in
spark history job app
Repository: incubator-eagle
Updated Branches:
refs/heads/master ce8e1c508 -> 4874433a5
[MINOR] add jobConf.additional.info in spark history job app
A quick fixed bug
Author: Zhao, Qingwen <qi...@apache.org>
Closes #662 from qingwen220/minor.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/4874433a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/4874433a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/4874433a
Branch: refs/heads/master
Commit: 4874433a59c59e599b73a1235819db92e3248ce1
Parents: ce8e1c5
Author: Zhao, Qingwen <qi...@apache.org>
Authored: Thu Nov 17 13:18:55 2016 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Thu Nov 17 13:18:55 2016 +0800
----------------------------------------------------------------------
.../eagle/jpm/spark/history/crawl/JHFSparkEventReader.java | 7 ++++++-
...che.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml | 6 ++++++
.../src/main/resources/application.conf | 1 +
3 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4874433a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java
index a4ac77e..82e8a41 100644
--- a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java
+++ b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java
@@ -115,10 +115,15 @@ public class JHFSparkEventReader {
app.setConfig(new JobConfig());
JSONObject sparkProps = (JSONObject) event.get("Spark Properties");
- String[] additionalJobConf = conf.getString("basic.jobConf.additional.info").split(",\\s*");
String[] props = {"spark.yarn.app.id", "spark.executor.memory", "spark.driver.host", "spark.driver.port",
"spark.driver.memory", "spark.scheduler.pool", "spark.executor.cores", "spark.yarn.am.memory",
"spark.yarn.am.cores", "spark.yarn.executor.memoryOverhead", "spark.yarn.driver.memoryOverhead", "spark.yarn.am.memoryOverhead", "spark.master"};
+
+ String[] additionalJobConf = null;
+ if (conf.hasPath("spark.jobConf.additional.info")) {
+ additionalJobConf = conf.getString("spark.jobConf.additional.info").split(",\\s*");
+ }
+
String[] jobConf = (String[]) ArrayUtils.addAll(additionalJobConf, props);
for (String prop : jobConf) {
if (sparkProps.containsKey(prop)) {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4874433a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml b/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml
index c68d4e8..17a3a4a 100644
--- a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml
+++ b/eagle-jpm/eagle-jpm-spark-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.spark.history.SparkHistoryJobAppProvider.xml
@@ -90,6 +90,12 @@
</property>
<property>
+ <name>spark.jobConf.additional.info</name>
+ <displayName>additional spark job config info</displayName>
+ <description>additional job config list</description>
+ <value></value>
+ </property>
+ <property>
<name>spark.defaultVal.spark.executor.memory</name>
<displayName>spark.executor.memory</displayName>
<value>1g</value>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4874433a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/application.conf b/eagle-jpm/eagle-jpm-spark-history/src/main/resources/application.conf
index a51abc9..2839915 100644
--- a/eagle-jpm/eagle-jpm-spark-history/src/main/resources/application.conf
+++ b/eagle-jpm/eagle-jpm-spark-history/src/main/resources/application.conf
@@ -57,6 +57,7 @@
numOfParserBoltTasks: 4
},
"spark":{
+ jobConf.additional.info: "",
"defaultVal": {
spark.executor.memory:"1g",
spark.driver.memory: "1g",