You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/07/17 02:31:12 UTC
[15/50] [abbrv] git commit: Determine Spark core classes better in
getCallSite
Determine Spark core classes better in getCallSite
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/a44a7b12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/a44a7b12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/a44a7b12
Branch: refs/heads/master
Commit: a44a7b123862202cf97f2de7a96aeaf29a93002a
Parents: e271fde
Author: Matei Zaharia <ma...@gmail.com>
Authored: Sun Jul 14 07:23:09 2013 +0000
Committer: Matei Zaharia <ma...@gmail.com>
Committed: Sun Jul 14 07:23:09 2013 +0000
----------------------------------------------------------------------
core/src/main/scala/spark/Utils.scala | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/a44a7b12/core/src/main/scala/spark/Utils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/spark/Utils.scala b/core/src/main/scala/spark/Utils.scala
index c6a3f97..a36186b 100644
--- a/core/src/main/scala/spark/Utils.scala
+++ b/core/src/main/scala/spark/Utils.scala
@@ -579,8 +579,15 @@ private object Utils extends Logging {
output.toString
}
+ /**
+ * A regular expression to match classes of the "core" Spark API that we want to skip when
+ * finding the call site of a method.
+ */
+ private val SPARK_CLASS_REGEX = """^spark(\.api\.java)?(\.rdd)?\.[A-Z]""".r
+
private[spark] class CallSiteInfo(val lastSparkMethod: String, val firstUserFile: String,
val firstUserLine: Int, val firstUserClass: String)
+
/**
* When called inside a class in the spark package, returns the name of the user code class
* (outside the spark package) that called into Spark, as well as which Spark method they called.
@@ -602,7 +609,7 @@ private object Utils extends Logging {
for (el <- trace) {
if (!finished) {
- if (el.getClassName.startsWith("spark.") && !el.getClassName.startsWith("spark.examples.")) {
+ if (SPARK_CLASS_REGEX.findFirstIn(el.getClassName) != None) {
lastSparkMethod = if (el.getMethodName == "<init>") {
// Spark method is a constructor; get its class name
el.getClassName.substring(el.getClassName.lastIndexOf('.') + 1)