You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by jo...@apache.org on 2015/06/29 21:04:01 UTC
spark git commit: [SPARK-8528] Expose SparkContext.applicationId in
PySpark
Repository: spark
Updated Branches:
refs/heads/master a5c2961ca -> 492dca3a7
[SPARK-8528] Expose SparkContext.applicationId in PySpark
Use case - we want to log applicationId (YARN in hour case) to request help with troubleshooting from the DevOps
Author: Vladimir Vladimirov <vl...@magnetic.com>
Closes #6936 from smartkiwi/master and squashes the following commits:
870338b [Vladimir Vladimirov] this would make doctest to run in python3
0eae619 [Vladimir Vladimirov] Scala doesn't use u'...' for unicode literals
14d77a8 [Vladimir Vladimirov] stop using ELLIPSIS
b4ebfc5 [Vladimir Vladimirov] addressed PR feedback - updated docstring
223a32f [Vladimir Vladimirov] fixed test - applicationId is property that returns the string
3221f5a [Vladimir Vladimirov] [SPARK-8528] added documentation for Scala
2cff090 [Vladimir Vladimirov] [SPARK-8528] add applicationId property for SparkContext object in pyspark
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/492dca3a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/492dca3a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/492dca3a
Branch: refs/heads/master
Commit: 492dca3a73e70705b5d5639e8fe4640b80e78d31
Parents: a5c2961
Author: Vladimir Vladimirov <vl...@magnetic.com>
Authored: Mon Jun 29 12:03:41 2015 -0700
Committer: Josh Rosen <jo...@databricks.com>
Committed: Mon Jun 29 12:03:41 2015 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/SparkContext.scala | 8 ++++++++
python/pyspark/context.py | 15 +++++++++++++++
2 files changed, 23 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/492dca3a/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index c7a7436..b3c3bf3 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -315,6 +315,14 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
_dagScheduler = ds
}
+ /**
+ * A unique identifier for the Spark application.
+ * Its format depends on the scheduler implementation.
+ * (i.e.
+ * in case of local spark app something like 'local-1433865536131'
+ * in case of YARN something like 'application_1433865536131_34483'
+ * )
+ */
def applicationId: String = _applicationId
def applicationAttemptId: Option[String] = _applicationAttemptId
http://git-wip-us.apache.org/repos/asf/spark/blob/492dca3a/python/pyspark/context.py
----------------------------------------------------------------------
diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index 90b2fff..d746672 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -292,6 +292,21 @@ class SparkContext(object):
return self._jsc.version()
@property
+ @ignore_unicode_prefix
+ def applicationId(self):
+ """
+ A unique identifier for the Spark application.
+ Its format depends on the scheduler implementation.
+ (i.e.
+ in case of local spark app something like 'local-1433865536131'
+ in case of YARN something like 'application_1433865536131_34483'
+ )
+ >>> sc.applicationId # doctest: +ELLIPSIS
+ u'local-...'
+ """
+ return self._jsc.sc().applicationId()
+
+ @property
def startTime(self):
"""Return the epoch time when the Spark Context was started."""
return self._jsc.startTime()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org