You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2017/02/20 20:21:14 UTC
spark git commit: [SPARK-19669][SQL] Open up visibility for
sharedState, sessionState, and a few other functions
Repository: spark
Updated Branches:
refs/heads/master ead4ba0eb -> 0733a54a4
[SPARK-19669][SQL] Open up visibility for sharedState, sessionState, and a few other functions
## What changes were proposed in this pull request?
To ease debugging, most of Spark SQL internals have public level visibility. Two of the most important internal states, sharedState and sessionState, however, are package private. It would make more sense to open these up as well with clear documentation that they are internal.
In addition, users currently have way to set active/default SparkSession, but no way to actually get them back. We should open those up as well.
## How was this patch tested?
N/A - only visibility change.
Author: Reynold Xin <rx...@databricks.com>
Closes #17002 from rxin/SPARK-19669.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0733a54a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0733a54a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0733a54a
Branch: refs/heads/master
Commit: 0733a54a4517b82291efed9ac7f7407d9044593c
Parents: ead4ba0
Author: Reynold Xin <rx...@databricks.com>
Authored: Mon Feb 20 12:21:07 2017 -0800
Committer: gatorsmile <ga...@gmail.com>
Committed: Mon Feb 20 12:21:07 2017 -0800
----------------------------------------------------------------------
.../org/apache/spark/internal/Logging.scala | 2 +-
.../org/apache/spark/sql/SparkSession.scala | 29 ++++++++++++++++----
.../apache/spark/sql/internal/SharedState.scala | 4 +--
3 files changed, 27 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0733a54a/core/src/main/scala/org/apache/spark/internal/Logging.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/internal/Logging.scala b/core/src/main/scala/org/apache/spark/internal/Logging.scala
index 013cd1c..c7f2847 100644
--- a/core/src/main/scala/org/apache/spark/internal/Logging.scala
+++ b/core/src/main/scala/org/apache/spark/internal/Logging.scala
@@ -28,7 +28,7 @@ import org.apache.spark.util.Utils
* logging messages at different levels using methods that only evaluate parameters lazily if the
* log level is enabled.
*/
-private[spark] trait Logging {
+trait Logging {
// Make the log field transient so that objects with Logging can
// be serialized and used on another machine
http://git-wip-us.apache.org/repos/asf/spark/blob/0733a54a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 1975a56..72af55c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -95,18 +95,28 @@ class SparkSession private(
/**
* State shared across sessions, including the `SparkContext`, cached data, listener,
* and a catalog that interacts with external systems.
+ *
+ * This is internal to Spark and there is no guarantee on interface stability.
+ *
+ * @since 2.2.0
*/
+ @InterfaceStability.Unstable
@transient
- private[sql] lazy val sharedState: SharedState = {
+ lazy val sharedState: SharedState = {
existingSharedState.getOrElse(new SharedState(sparkContext))
}
/**
* State isolated across sessions, including SQL configurations, temporary tables, registered
* functions, and everything else that accepts a [[org.apache.spark.sql.internal.SQLConf]].
+ *
+ * This is internal to Spark and there is no guarantee on interface stability.
+ *
+ * @since 2.2.0
*/
+ @InterfaceStability.Unstable
@transient
- private[sql] lazy val sessionState: SessionState = {
+ lazy val sessionState: SessionState = {
SparkSession.reflect[SessionState, SparkSession](
SparkSession.sessionStateClassName(sparkContext.conf),
self)
@@ -613,7 +623,6 @@ class SparkSession private(
*
* @since 2.1.0
*/
- @InterfaceStability.Stable
def time[T](f: => T): T = {
val start = System.nanoTime()
val ret = f
@@ -928,9 +937,19 @@ object SparkSession {
defaultSession.set(null)
}
- private[sql] def getActiveSession: Option[SparkSession] = Option(activeThreadSession.get)
+ /**
+ * Returns the active SparkSession for the current thread, returned by the builder.
+ *
+ * @since 2.2.0
+ */
+ def getActiveSession: Option[SparkSession] = Option(activeThreadSession.get)
- private[sql] def getDefaultSession: Option[SparkSession] = Option(defaultSession.get)
+ /**
+ * Returns the default SparkSession that is returned by the builder.
+ *
+ * @since 2.2.0
+ */
+ def getDefaultSession: Option[SparkSession] = Option(defaultSession.get)
/** A global SQL listener used for the SQL UI. */
private[sql] val sqlListener = new AtomicReference[SQLListener]()
http://git-wip-us.apache.org/repos/asf/spark/blob/0733a54a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
index 8de95fe..7ce9938 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
@@ -39,7 +39,7 @@ private[sql] class SharedState(val sparkContext: SparkContext) extends Logging {
// Load hive-site.xml into hadoopConf and determine the warehouse path we want to use, based on
// the config from both hive and Spark SQL. Finally set the warehouse config value to sparkConf.
- val warehousePath = {
+ val warehousePath: String = {
val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml")
if (configFile != null) {
sparkContext.hadoopConfiguration.addResource(configFile)
@@ -103,7 +103,7 @@ private[sql] class SharedState(val sparkContext: SparkContext) extends Logging {
/**
* A manager for global temporary views.
*/
- val globalTempViewManager = {
+ val globalTempViewManager: GlobalTempViewManager = {
// System preserved database should not exists in metastore. However it's hard to guarantee it
// for every session, because case-sensitivity differs. Here we always lowercase it to make our
// life easier.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org