You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/01/02 21:57:50 UTC
[3/4] git commit: Merge pull request #311 from tmyklebu/master
Merge pull request #311 from tmyklebu/master
SPARK-991: Report information gleaned from a Python stacktrace in the UI
Scala:
- Added setCallSite/clearCallSite to SparkContext and JavaSparkContext.
These functions mutate a LocalProperty called "externalCallSite."
- Add a wrapper, getCallSite, that checks for an externalCallSite and, if
none is found, calls the usual Utils.formatSparkCallSite.
- Change everything that calls Utils.formatSparkCallSite to call
getCallSite instead. Except getCallSite.
- Add wrappers to setCallSite/clearCallSite wrappers to JavaSparkContext.
Python:
- Add a gruesome hack to rdd.py that inspects the traceback and guesses
what you want to see in the UI.
- Add a RAII wrapper around said gruesome hack that calls
setCallSite/clearCallSite as appropriate.
- Wire said RAII wrapper up around three calls into the Scala code.
I'm not sure that I hit all the spots with the RAII wrapper. I'm also
not sure that my gruesome hack does exactly what we want.
One could also approach this change by refactoring
runJob/submitJob/runApproximateJob to take a call site, then threading
that parameter through everything that needs to know it.
One might object to the pointless-looking wrappers in JavaSparkContext.
Unfortunately, I can't directly access the SparkContext from
Python---or, if I can, I don't know how---so I need to wrap everything
that matters in JavaSparkContext.
Conflicts:
core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/ca67909c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/ca67909c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/ca67909c
Branch: refs/heads/master
Commit: ca67909cd4b4accea64498b3bc1421c6d75e33a6
Parents: 3713f81 fec0166
Author: Matei Zaharia <ma...@databricks.com>
Authored: Thu Jan 2 15:54:54 2014 -0500
Committer: Matei Zaharia <ma...@databricks.com>
Committed: Thu Jan 2 15:54:54 2014 -0500
----------------------------------------------------------------------
.../scala/org/apache/spark/SparkContext.scala | 26 +++++++-
.../spark/api/java/JavaSparkContext.scala | 14 +++++
.../main/scala/org/apache/spark/rdd/RDD.scala | 2 +-
python/pyspark/rdd.py | 66 ++++++++++++++++----
4 files changed, 93 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/ca67909c/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/ca67909c/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
----------------------------------------------------------------------
diff --cc core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
index 0680a06,3003e7a..5be5317
--- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
@@@ -407,10 -407,18 +407,24 @@@ class JavaSparkContext(val sc: SparkCon
}
/**
+ * Return a copy of this JavaSparkContext's configuration. The configuration ''cannot'' be
+ * changed at runtime.
+ */
+ def getConf: SparkConf = sc.getConf
++
++ /**
+ * Pass-through to SparkContext.setCallSite. For API support only.
+ */
+ def setCallSite(site: String) {
+ sc.setCallSite(site)
+ }
+
+ /**
+ * Pass-through to SparkContext.setCallSite. For API support only.
+ */
+ def clearCallSite() {
+ sc.clearCallSite()
+ }
}
object JavaSparkContext {
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/ca67909c/core/src/main/scala/org/apache/spark/rdd/RDD.scala
----------------------------------------------------------------------