You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ir...@apache.org on 2017/11/07 22:03:33 UTC
spark git commit: [SPARK-20645][CORE] Port environment page to new UI
backend.
Repository: spark
Updated Branches:
refs/heads/master 0846a4473 -> 7475a9655
[SPARK-20645][CORE] Port environment page to new UI backend.
This change modifies the status listener to collect the information
needed to render the envionment page, and populates that page and the
API with information collected by the listener.
Tested with existing and added unit tests.
Author: Marcelo Vanzin <va...@cloudera.com>
Closes #19677 from vanzin/SPARK-20645.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7475a965
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7475a965
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7475a965
Branch: refs/heads/master
Commit: 7475a9655cd23dbf667e9543cf0818243e59f998
Parents: 0846a44
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Tue Nov 7 16:03:24 2017 -0600
Committer: Imran Rashid <ir...@cloudera.com>
Committed: Tue Nov 7 16:03:24 2017 -0600
----------------------------------------------------------------------
.../apache/spark/status/AppStatusListener.scala | 18 ++
.../apache/spark/status/AppStatusStore.scala | 9 +
.../api/v1/ApplicationEnvironmentResource.scala | 15 +-
.../org/apache/spark/status/storeTypes.scala | 11 +
.../scala/org/apache/spark/ui/SparkUI.scala | 28 +-
.../apache/spark/ui/env/EnvironmentPage.scala | 31 +-
.../apache/spark/ui/env/EnvironmentTab.scala | 56 ----
.../app_environment_expectation.json | 281 +++++++++++++++++++
.../deploy/history/HistoryServerSuite.scala | 4 +-
.../spark/status/AppStatusListenerSuite.scala | 40 +++
project/MimaExcludes.scala | 1 +
11 files changed, 402 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/7475a965/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index cd43612..424a115 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -88,6 +88,24 @@ private[spark] class AppStatusListener(
kvstore.write(new ApplicationInfoWrapper(appInfo))
}
+ override def onEnvironmentUpdate(event: SparkListenerEnvironmentUpdate): Unit = {
+ val details = event.environmentDetails
+
+ val jvmInfo = Map(details("JVM Information"): _*)
+ val runtime = new v1.RuntimeInfo(
+ jvmInfo.get("Java Version").orNull,
+ jvmInfo.get("Java Home").orNull,
+ jvmInfo.get("Scala Version").orNull)
+
+ val envInfo = new v1.ApplicationEnvironmentInfo(
+ runtime,
+ details.getOrElse("Spark Properties", Nil),
+ details.getOrElse("System Properties", Nil),
+ details.getOrElse("Classpath Entries", Nil))
+
+ kvstore.write(new ApplicationEnvironmentInfoWrapper(envInfo))
+ }
+
override def onApplicationEnd(event: SparkListenerApplicationEnd): Unit = {
val old = appInfo.attempts.head
val attempt = new v1.ApplicationAttemptInfo(
http://git-wip-us.apache.org/repos/asf/spark/blob/7475a965/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
index 2927a32..d6b5d26 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
@@ -33,6 +33,15 @@ import org.apache.spark.util.kvstore.{InMemoryStore, KVStore}
*/
private[spark] class AppStatusStore(store: KVStore) {
+ def applicationInfo(): v1.ApplicationInfo = {
+ store.view(classOf[ApplicationInfoWrapper]).max(1).iterator().next().info
+ }
+
+ def environmentInfo(): v1.ApplicationEnvironmentInfo = {
+ val klass = classOf[ApplicationEnvironmentInfoWrapper]
+ store.read(klass, klass.getName()).info
+ }
+
def jobsList(statuses: JList[JobExecutionStatus]): Seq[v1.JobData] = {
val it = store.view(classOf[JobDataWrapper]).asScala.map(_.info)
if (!statuses.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/spark/blob/7475a965/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationEnvironmentResource.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationEnvironmentResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationEnvironmentResource.scala
index 739a8ac..e702f8a 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationEnvironmentResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationEnvironmentResource.scala
@@ -26,20 +26,7 @@ private[v1] class ApplicationEnvironmentResource(ui: SparkUI) {
@GET
def getEnvironmentInfo(): ApplicationEnvironmentInfo = {
- val listener = ui.environmentListener
- listener.synchronized {
- val jvmInfo = Map(listener.jvmInformation: _*)
- val runtime = new RuntimeInfo(
- jvmInfo("Java Version"),
- jvmInfo("Java Home"),
- jvmInfo("Scala Version"))
-
- new ApplicationEnvironmentInfo(
- runtime,
- listener.sparkProperties,
- listener.systemProperties,
- listener.classpathEntries)
- }
+ ui.store.environmentInfo()
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/7475a965/core/src/main/scala/org/apache/spark/status/storeTypes.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/status/storeTypes.scala b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
index a445435..23e9a36 100644
--- a/core/src/main/scala/org/apache/spark/status/storeTypes.scala
+++ b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
@@ -34,6 +34,17 @@ private[spark] class ApplicationInfoWrapper(val info: ApplicationInfo) {
}
+private[spark] class ApplicationEnvironmentInfoWrapper(val info: ApplicationEnvironmentInfo) {
+
+ /**
+ * There's always a single ApplicationEnvironmentInfo object per application, so this
+ * ID doesn't need to be dynamic. But the KVStore API requires an ID.
+ */
+ @JsonIgnore @KVIndex
+ def id: String = classOf[ApplicationEnvironmentInfoWrapper].getName()
+
+}
+
private[spark] class ExecutorSummaryWrapper(val info: ExecutorSummary) {
@JsonIgnore @KVIndex
http://git-wip-us.apache.org/repos/asf/spark/blob/7475a965/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
index ee645f6..43b57a1 100644
--- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -25,11 +25,10 @@ import org.apache.spark.{SecurityManager, SparkConf, SparkContext}
import org.apache.spark.internal.Logging
import org.apache.spark.scheduler._
import org.apache.spark.status.AppStatusStore
-import org.apache.spark.status.api.v1.{ApiRootResource, ApplicationAttemptInfo, ApplicationInfo,
- UIRoot}
+import org.apache.spark.status.api.v1._
import org.apache.spark.storage.StorageStatusListener
import org.apache.spark.ui.JettyUtils._
-import org.apache.spark.ui.env.{EnvironmentListener, EnvironmentTab}
+import org.apache.spark.ui.env.EnvironmentTab
import org.apache.spark.ui.exec.{ExecutorsListener, ExecutorsTab}
import org.apache.spark.ui.jobs.{JobProgressListener, JobsTab, StagesTab}
import org.apache.spark.ui.scope.RDDOperationGraphListener
@@ -44,7 +43,6 @@ private[spark] class SparkUI private (
val sc: Option[SparkContext],
val conf: SparkConf,
securityManager: SecurityManager,
- val environmentListener: EnvironmentListener,
val storageStatusListener: StorageStatusListener,
val executorsListener: ExecutorsListener,
val jobProgressListener: JobProgressListener,
@@ -73,7 +71,7 @@ private[spark] class SparkUI private (
val stagesTab = new StagesTab(this)
attachTab(stagesTab)
attachTab(new StorageTab(this))
- attachTab(new EnvironmentTab(this))
+ attachTab(new EnvironmentTab(this, store))
attachTab(new ExecutorsTab(this))
attachHandler(createStaticHandler(SparkUI.STATIC_RESOURCE_DIR, "/static"))
attachHandler(createRedirectHandler("/", "/jobs/", basePath = basePath))
@@ -88,9 +86,13 @@ private[spark] class SparkUI private (
initialize()
def getSparkUser: String = {
- environmentListener.sparkUser
- .orElse(environmentListener.systemProperties.toMap.get("user.name"))
- .getOrElse("<unknown>")
+ try {
+ Option(store.applicationInfo().attempts.head.sparkUser)
+ .orElse(store.environmentInfo().systemProperties.toMap.get("user.name"))
+ .getOrElse("<unknown>")
+ } catch {
+ case _: NoSuchElementException => "<unknown>"
+ }
}
def getAppName: String = appName
@@ -143,6 +145,7 @@ private[spark] class SparkUI private (
def setStreamingJobProgressListener(sparkListener: SparkListener): Unit = {
streamingJobProgressListener = Option(sparkListener)
}
+
}
private[spark] abstract class SparkUITab(parent: SparkUI, prefix: String)
@@ -184,21 +187,20 @@ private[spark] object SparkUI {
addListenerFn(listener)
listener
}
- val environmentListener = new EnvironmentListener
+
val storageStatusListener = new StorageStatusListener(conf)
val executorsListener = new ExecutorsListener(storageStatusListener, conf)
val storageListener = new StorageListener(storageStatusListener)
val operationGraphListener = new RDDOperationGraphListener(conf)
- addListenerFn(environmentListener)
addListenerFn(storageStatusListener)
addListenerFn(executorsListener)
addListenerFn(storageListener)
addListenerFn(operationGraphListener)
- new SparkUI(store, sc, conf, securityManager, environmentListener, storageStatusListener,
- executorsListener, jobProgressListener, storageListener, operationGraphListener,
- appName, basePath, lastUpdateTime, startTime, appSparkVersion)
+ new SparkUI(store, sc, conf, securityManager, storageStatusListener, executorsListener,
+ jobProgressListener, storageListener, operationGraphListener, appName, basePath,
+ lastUpdateTime, startTime, appSparkVersion)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/7475a965/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
index b11f8f1..43adab7a 100644
--- a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
@@ -21,22 +21,31 @@ import javax.servlet.http.HttpServletRequest
import scala.xml.Node
-import org.apache.spark.ui.{UIUtils, WebUIPage}
+import org.apache.spark.SparkConf
+import org.apache.spark.status.AppStatusStore
+import org.apache.spark.ui._
import org.apache.spark.util.Utils
-private[ui] class EnvironmentPage(parent: EnvironmentTab) extends WebUIPage("") {
- private val listener = parent.listener
+private[ui] class EnvironmentPage(
+ parent: EnvironmentTab,
+ conf: SparkConf,
+ store: AppStatusStore) extends WebUIPage("") {
def render(request: HttpServletRequest): Seq[Node] = {
+ val appEnv = store.environmentInfo()
+ val jvmInformation = Map(
+ "Java Version" -> appEnv.runtime.javaVersion,
+ "Java Home" -> appEnv.runtime.javaHome,
+ "Scala Version" -> appEnv.runtime.scalaVersion)
+
val runtimeInformationTable = UIUtils.listingTable(
- propertyHeader, jvmRow, listener.jvmInformation, fixedWidth = true)
+ propertyHeader, jvmRow, jvmInformation, fixedWidth = true)
val sparkPropertiesTable = UIUtils.listingTable(propertyHeader, propertyRow,
- Utils.redact(parent.conf, listener.sparkProperties), fixedWidth = true)
-
+ Utils.redact(conf, appEnv.sparkProperties.toSeq), fixedWidth = true)
val systemPropertiesTable = UIUtils.listingTable(
- propertyHeader, propertyRow, listener.systemProperties, fixedWidth = true)
+ propertyHeader, propertyRow, appEnv.systemProperties, fixedWidth = true)
val classpathEntriesTable = UIUtils.listingTable(
- classPathHeaders, classPathRow, listener.classpathEntries, fixedWidth = true)
+ classPathHeaders, classPathRow, appEnv.classpathEntries, fixedWidth = true)
val content =
<span>
<h4>Runtime Information</h4> {runtimeInformationTable}
@@ -54,3 +63,9 @@ private[ui] class EnvironmentPage(parent: EnvironmentTab) extends WebUIPage("")
private def propertyRow(kv: (String, String)) = <tr><td>{kv._1}</td><td>{kv._2}</td></tr>
private def classPathRow(data: (String, String)) = <tr><td>{data._1}</td><td>{data._2}</td></tr>
}
+
+private[ui] class EnvironmentTab(
+ parent: SparkUI,
+ store: AppStatusStore) extends SparkUITab(parent, "environment") {
+ attachPage(new EnvironmentPage(this, parent.conf, store))
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/7475a965/core/src/main/scala/org/apache/spark/ui/env/EnvironmentTab.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentTab.scala b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentTab.scala
deleted file mode 100644
index 61b12aa..0000000
--- a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentTab.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.ui.env
-
-import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.scheduler._
-import org.apache.spark.ui._
-
-private[ui] class EnvironmentTab(parent: SparkUI) extends SparkUITab(parent, "environment") {
- val listener = parent.environmentListener
- val conf = parent.conf
- attachPage(new EnvironmentPage(this))
-}
-
-/**
- * :: DeveloperApi ::
- * A SparkListener that prepares information to be displayed on the EnvironmentTab
- */
-@DeveloperApi
-@deprecated("This class will be removed in a future release.", "2.2.0")
-class EnvironmentListener extends SparkListener {
- var sparkUser: Option[String] = None
- var jvmInformation = Seq[(String, String)]()
- var sparkProperties = Seq[(String, String)]()
- var systemProperties = Seq[(String, String)]()
- var classpathEntries = Seq[(String, String)]()
-
- override def onApplicationStart(event: SparkListenerApplicationStart): Unit = {
- sparkUser = Some(event.sparkUser)
- }
-
- override def onEnvironmentUpdate(environmentUpdate: SparkListenerEnvironmentUpdate) {
- synchronized {
- val environmentDetails = environmentUpdate.environmentDetails
- jvmInformation = environmentDetails("JVM Information")
- sparkProperties = environmentDetails("Spark Properties")
- systemProperties = environmentDetails("System Properties")
- classpathEntries = environmentDetails("Classpath Entries")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/7475a965/core/src/test/resources/HistoryServerExpectations/app_environment_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/app_environment_expectation.json b/core/src/test/resources/HistoryServerExpectations/app_environment_expectation.json
new file mode 100644
index 0000000..4ed0538
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/app_environment_expectation.json
@@ -0,0 +1,281 @@
+{
+ "runtime" : {
+ "javaVersion" : "1.8.0_92 (Oracle Corporation)",
+ "javaHome" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre",
+ "scalaVersion" : "version 2.11.8"
+ },
+ "sparkProperties" : [
+ [ "spark.blacklist.task.maxTaskAttemptsPerExecutor", "3" ],
+ [ "spark.blacklist.enabled", "TRUE" ],
+ [ "spark.driver.host", "172.22.0.167" ],
+ [ "spark.blacklist.task.maxTaskAttemptsPerNode", "3" ],
+ [ "spark.eventLog.enabled", "TRUE" ],
+ [ "spark.driver.port", "51459" ],
+ [ "spark.repl.class.uri", "spark://172.22.0.167:51459/classes" ],
+ [ "spark.jars", "" ],
+ [ "spark.repl.class.outputDir", "/private/var/folders/l4/d46wlzj16593f3d812vk49tw0000gp/T/spark-1cbc97d0-7fe6-4c9f-8c2c-f6fe51ee3cf2/repl-39929169-ac4c-4c6d-b116-f648e4dd62ed" ],
+ [ "spark.app.name", "Spark shell" ],
+ [ "spark.blacklist.stage.maxFailedExecutorsPerNode", "3" ],
+ [ "spark.scheduler.mode", "FIFO" ],
+ [ "spark.eventLog.overwrite", "TRUE" ],
+ [ "spark.blacklist.stage.maxFailedTasksPerExecutor", "3" ],
+ [ "spark.executor.id", "driver" ],
+ [ "spark.blacklist.application.maxFailedExecutorsPerNode", "2" ],
+ [ "spark.submit.deployMode", "client" ],
+ [ "spark.master", "local-cluster[4,4,1024]" ],
+ [ "spark.home", "/Users/Jose/IdeaProjects/spark" ],
+ [ "spark.eventLog.dir", "/Users/jose/logs" ],
+ [ "spark.sql.catalogImplementation", "in-memory" ],
+ [ "spark.eventLog.compress", "FALSE" ],
+ [ "spark.blacklist.application.maxFailedTasksPerExecutor", "1" ],
+ [ "spark.blacklist.timeout", "1000000" ],
+ [ "spark.app.id", "app-20161116163331-0000" ],
+ [ "spark.task.maxFailures", "4" ]
+ ],
+ "systemProperties" : [
+ [ "java.io.tmpdir", "/var/folders/l4/d46wlzj16593f3d812vk49tw0000gp/T/" ],
+ [ "line.separator", "\n" ],
+ [ "path.separator", ":" ],
+ [ "sun.management.compiler", "HotSpot 64-Bit Tiered Compilers" ],
+ [ "SPARK_SUBMIT", "true" ],
+ [ "sun.cpu.endian", "little" ],
+ [ "java.specification.version", "1.8" ],
+ [ "java.vm.specification.name", "Java Virtual Machine Specification" ],
+ [ "java.vendor", "Oracle Corporation" ],
+ [ "java.vm.specification.version", "1.8" ],
+ [ "user.home", "/Users/Jose" ],
+ [ "file.encoding.pkg", "sun.io" ],
+ [ "sun.nio.ch.bugLevel", "" ],
+ [ "ftp.nonProxyHosts", "local|*.local|169.254/16|*.169.254/16" ],
+ [ "sun.arch.data.model", "64" ],
+ [ "sun.boot.library.path", "/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib" ],
+ [ "user.dir", "/Users/Jose/IdeaProjects/spark" ],
+ [ "java.library.path", "/Users/Jose/Library/Java/Extensions:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java:." ],
+ [ "sun.cpu.isalist", "" ],
+ [ "os.arch", "x86_64" ],
+ [ "java.vm.version", "25.92-b14" ],
+ [ "java.endorsed.dirs", "/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/endorsed" ],
+ [ "java.runtime.version", "1.8.0_92-b14" ],
+ [ "java.vm.info", "mixed mode" ],
+ [ "java.ext.dirs", "/Users/Jose/Library/Java/Extensions:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/ext:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java" ],
+ [ "java.runtime.name", "Java(TM) SE Runtime Environment" ],
+ [ "file.separator", "/" ],
+ [ "io.netty.maxDirectMemory", "0" ],
+ [ "java.class.version", "52.0" ],
+ [ "scala.usejavacp", "true" ],
+ [ "java.specification.name", "Java Platform API Specification" ],
+ [ "sun.boot.class.path", "/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/sunrsasign.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre/classes" ],
+ [ "file.encoding", "UTF-8" ],
+ [ "user.timezone", "America/Chicago" ],
+ [ "java.specification.vendor", "Oracle Corporation" ],
+ [ "sun.java.launcher", "SUN_STANDARD" ],
+ [ "os.version", "10.11.6" ],
+ [ "sun.os.patch.level", "unknown" ],
+ [ "gopherProxySet", "false" ],
+ [ "java.vm.specification.vendor", "Oracle Corporation" ],
+ [ "user.country", "US" ],
+ [ "sun.jnu.encoding", "UTF-8" ],
+ [ "http.nonProxyHosts", "local|*.local|169.254/16|*.169.254/16" ],
+ [ "user.language", "en" ],
+ [ "socksNonProxyHosts", "local|*.local|169.254/16|*.169.254/16" ],
+ [ "java.vendor.url", "http://java.oracle.com/" ],
+ [ "java.awt.printerjob", "sun.lwawt.macosx.CPrinterJob" ],
+ [ "java.awt.graphicsenv", "sun.awt.CGraphicsEnvironment" ],
+ [ "awt.toolkit", "sun.lwawt.macosx.LWCToolkit" ],
+ [ "os.name", "Mac OS X" ],
+ [ "java.vm.vendor", "Oracle Corporation" ],
+ [ "java.vendor.url.bug", "http://bugreport.sun.com/bugreport/" ],
+ [ "user.name", "jose" ],
+ [ "java.vm.name", "Java HotSpot(TM) 64-Bit Server VM" ],
+ [ "sun.java.command", "org.apache.spark.deploy.SparkSubmit --master local-cluster[4,4,1024] --conf spark.blacklist.enabled=TRUE --conf spark.blacklist.timeout=1000000 --conf spark.blacklist.application.maxFailedTasksPerExecutor=1 --conf spark.eventLog.overwrite=TRUE --conf spark.blacklist.task.maxTaskAttemptsPerNode=3 --conf spark.blacklist.stage.maxFailedTasksPerExecutor=3 --conf spark.blacklist.task.maxTaskAttemptsPerExecutor=3 --conf spark.eventLog.compress=FALSE --conf spark.blacklist.stage.maxFailedExecutorsPerNode=3 --conf spark.eventLog.enabled=TRUE --conf spark.eventLog.dir=/Users/jose/logs --conf spark.blacklist.application.maxFailedExecutorsPerNode=2 --conf spark.task.maxFailures=4 --class org.apache.spark.repl.Main --name Spark shell spark-shell -i /Users/Jose/dev/jose-utils/blacklist/test-blacklist.scala" ],
+ [ "java.home", "/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre" ],
+ [ "java.version", "1.8.0_92" ],
+ [ "sun.io.unicode.encoding", "UnicodeBig" ]
+ ],
+ "classpathEntries" : [
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/avro-mapred-1.7.7-hadoop2.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-mapreduce-client-core-2.2.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-servlet-9.2.16.v20160414.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-column-1.8.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/snappy-java-1.1.2.6.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/oro-2.0.8.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/arpack_combined_all-0.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/pmml-schema-1.2.15.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-assembly_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javassist-3.18.1-GA.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-tags_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-launcher_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-math3-3.4.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hk2-api-2.4.0-b34.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scala-xml_2.11-1.0.4.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/objenesis-2.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spire-macros_2.11-0.7.4.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scala-reflect-2.11.8.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-mllib-local_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-mllib_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-server-2.22.2.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/core/target/scala-2.11/classes/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-mapper-asl-1.9.13.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-module-scala_2.11-2.6.5.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/curator-framework-2.4.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javax.inject-1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/curator-client-2.4.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-core-asl-1.9.13.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/common/network-common/target/scala-2.11/classes/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/zookeeper-3.4.5.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-auth-2.2.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/repl/target/scala-2.11/classes/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jul-to-slf4j-1.7.16.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-media-jaxb-2.22.2.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-io-9.2.16.v20160414.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/RoaringBitmap-0.5.11.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javax.ws.rs-api-2.0.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/sql/catalyst/target/scala-2.11/classes/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-unsafe_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-repl_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-continuation-9.2.16.v20160414.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-yarn-client-2.2.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/sql/hive-thriftserver/target/scala-2.11/classes", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-annotations-2.2.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/metrics-graphite-3.1.2.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-yarn-api-2.2.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-container-servlet-core-2.22.2.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/streaming/target/scala-2.11/classes/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-net-3.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-proxy-9.2.16.v20160414.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-catalyst_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/lz4-1.3.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-crypto-1.0.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/common/network-yarn/target/scala-2.11/classes", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javax.annotation-api-1.2.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-sql_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/guava-14.0.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javax.servlet-api-3.1.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-collections-3.2.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/conf/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/unused-1.0.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/aopalliance-1.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-encoding-1.8.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/common/tags/target/scala-2.11/classes/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/json4s-jackson_2.11-3.2.11.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-cli-1.2.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-yarn-server-common-2.2.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/cglib-2.2.1-v20090111.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/pyrolite-4.13.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scala-library-2.11.8.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scala-parser-combinators_2.11-1.0.4.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-util-6.1.26.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/py4j-0.10.4.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-configuration-1.6.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/core-1.1.2.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/core/target/jars/*", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/common/network-shuffle/target/scala-2.11/classes/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-format-2.3.0-incubating.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/kryo-shaded-3.0.3.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/sql/core/target/scala-2.11/classes/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/chill-java-0.8.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-annotations-2.6.5.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-hadoop-1.8.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/sql/hive/target/scala-2.11/classes/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/avro-ipc-1.7.7.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/xz-1.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-jackson-1.8.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/aopalliance-repackaged-2.4.0-b34.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-common-2.22.2.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/log4j-1.2.17.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/metrics-core-3.1.2.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-util-9.2.16.v20160414.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scalap-2.11.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/osgi-resource-locator-1.0.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-beanutils-1.7.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-compress-1.4.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jcl-over-slf4j-1.7.16.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/yarn/target/scala-2.11/classes", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-plus-9.2.16.v20160414.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/protobuf-java-2.5.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/common/unsafe/target/scala-2.11/classes/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-module-paranamer-2.6.5.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/leveldbjni-all-1.8.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-core-2.6.5.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/slf4j-api-1.7.16.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/compress-lzf-1.0.3.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/stream-2.7.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-mapreduce-client-shuffle-2.2.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-codec-1.10.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-yarn-common-2.2.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/common/sketch/target/scala-2.11/classes/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/breeze_2.11-0.12.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-mapreduce-client-common-2.2.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-core_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-container-servlet-2.22.2.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-network-shuffle_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-lang-2.5.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/ivy-2.4.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-common-2.2.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-math-2.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-hdfs-2.2.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scala-compiler-2.11.8.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/metrics-jvm-3.1.2.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-lang3-3.5.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jsr305-1.3.9.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/minlog-1.3.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/netty-3.8.0.Final.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-webapp-9.2.16.v20160414.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/json4s-ast_2.11-3.2.11.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/xbean-asm5-shaded-4.4.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-io-2.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/slf4j-log4j12-1.7.16.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hk2-locator-2.4.0-b34.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/shapeless_2.11-2.0.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-network-common_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-xml-9.2.16.v20160414.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-httpclient-3.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/javax.inject-2.4.0-b34.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/mllib/target/scala-2.11/classes/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/scalatest_2.11-2.2.6.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hk2-utils-2.4.0-b34.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-client-9.2.16.v20160414.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-guava-2.22.2.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-jndi-9.2.16.v20160414.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/graphx/target/scala-2.11/classes/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-mapreduce-client-app-2.2.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/examples/target/scala-2.11/classes/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/xmlenc-0.52.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jets3t-0.7.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/curator-recipes-2.4.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/opencsv-2.3.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jtransforms-2.4.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/antlr4-runtime-4.5.3.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/chill_2.11-0.8.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-digester-1.8.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/univocity-parsers-2.2.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jline-2.12.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-streaming_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/launcher/target/scala-2.11/classes/", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/breeze-macros_2.11-0.12.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jersey-client-2.22.2.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jackson-databind-2.6.5.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-servlets-9.2.16.v20160414.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/paranamer-2.6.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-security-9.2.16.v20160414.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/avro-ipc-1.7.7-tests.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/avro-1.7.7.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spire_2.11-0.7.4.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-client-2.2.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/metrics-json-3.1.2.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-beanutils-core-1.8.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/validation-api-1.1.0.Final.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-graphx_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/netty-all-4.0.41.Final.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/janino-3.0.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/json4s-core_2.11-3.2.11.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/commons-compiler-3.0.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/guice-3.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-server-9.2.16.v20160414.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/jetty-http-9.2.16.v20160414.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/parquet-common-1.8.1.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/hadoop-mapreduce-client-jobclient-2.2.0.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/spark-sketch_2.11-2.1.0-SNAPSHOT.jar", "System Classpath" ],
+ [ "/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/pmml-model-1.2.15.jar", "System Classpath" ]
+ ]
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/7475a965/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index 010a8dd..6a1abce 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -158,7 +158,9 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
"rdd list storage json" -> "applications/local-1422981780767/storage/rdd",
"executor node blacklisting" -> "applications/app-20161116163331-0000/executors",
"executor node blacklisting unblacklisting" -> "applications/app-20161115172038-0000/executors",
- "executor memory usage" -> "applications/app-20161116163331-0000/executors"
+ "executor memory usage" -> "applications/app-20161116163331-0000/executors",
+
+ "app environment" -> "applications/app-20161116163331-0000/environment"
// Todo: enable this test when logging the even of onBlockUpdated. See: SPARK-13845
// "one rdd storage json" -> "applications/local-1422981780767/storage/rdd/0"
)
http://git-wip-us.apache.org/repos/asf/spark/blob/7475a965/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
index 7ac1ce1..867d35f 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
@@ -56,6 +56,46 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
Utils.deleteRecursively(testDir)
}
+ test("environment info") {
+ val listener = new AppStatusListener(store, conf, true)
+
+ val details = Map(
+ "JVM Information" -> Seq(
+ "Java Version" -> sys.props("java.version"),
+ "Java Home" -> sys.props("java.home"),
+ "Scala Version" -> scala.util.Properties.versionString
+ ),
+ "Spark Properties" -> Seq(
+ "spark.conf.1" -> "1",
+ "spark.conf.2" -> "2"
+ ),
+ "System Properties" -> Seq(
+ "sys.prop.1" -> "1",
+ "sys.prop.2" -> "2"
+ ),
+ "Classpath Entries" -> Seq(
+ "/jar1" -> "System",
+ "/jar2" -> "User"
+ )
+ )
+
+ listener.onEnvironmentUpdate(SparkListenerEnvironmentUpdate(details))
+
+ val appEnvKey = classOf[ApplicationEnvironmentInfoWrapper].getName()
+ check[ApplicationEnvironmentInfoWrapper](appEnvKey) { env =>
+ val info = env.info
+
+ val runtimeInfo = Map(details("JVM Information"): _*)
+ assert(info.runtime.javaVersion == runtimeInfo("Java Version"))
+ assert(info.runtime.javaHome == runtimeInfo("Java Home"))
+ assert(info.runtime.scalaVersion == runtimeInfo("Scala Version"))
+
+ assert(info.sparkProperties === details("Spark Properties"))
+ assert(info.systemProperties === details("System Properties"))
+ assert(info.classpathEntries === details("Classpath Entries"))
+ }
+ }
+
test("scheduler events") {
val listener = new AppStatusListener(store, conf, true)
http://git-wip-us.apache.org/repos/asf/spark/blob/7475a965/project/MimaExcludes.scala
----------------------------------------------------------------------
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 99cac34..62930e2 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -39,6 +39,7 @@ object MimaExcludes {
// SPARK-18085: Better History Server scalability for many / large applications
ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.ExecutorSummary.executorLogs"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.deploy.history.HistoryServer.getSparkUI"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.ui.env.EnvironmentListener"),
// [SPARK-20495][SQL] Add StorageLevel to cacheTable API
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.cacheTable"),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org