You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2016/05/09 18:25:04 UTC
spark git commit: [SPARK-15166][SQL] Move some hive-specific code
from SparkSession
Repository: spark
Updated Branches:
refs/heads/master dfdcab00c -> 7bf9b1201
[SPARK-15166][SQL] Move some hive-specific code from SparkSession
## What changes were proposed in this pull request?
This also simplifies the code being moved.
## How was this patch tested?
Existing tests.
Author: Andrew Or <an...@databricks.com>
Closes #12941 from andrewor14/move-code.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7bf9b120
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7bf9b120
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7bf9b120
Branch: refs/heads/master
Commit: 7bf9b12019bb20470b726a7233d60ce38a9c52cc
Parents: dfdcab0
Author: Andrew Or <an...@databricks.com>
Authored: Mon May 9 11:24:58 2016 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Mon May 9 11:24:58 2016 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/SparkSession.scala | 13 -------------
.../apache/spark/sql/hive/HiveSessionState.scala | 2 --
.../apache/spark/sql/hive/HiveSharedState.scala | 17 +++++++++++++----
3 files changed, 13 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/7bf9b120/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 2a893c6..c7fa8f7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -73,19 +73,6 @@ class SparkSession private(
| Session-related state |
* ----------------------- */
- {
- val defaultWarehousePath =
- SQLConf.WAREHOUSE_PATH
- .defaultValueString
- .replace("${system:user.dir}", System.getProperty("user.dir"))
- val warehousePath = sparkContext.conf.get(
- SQLConf.WAREHOUSE_PATH.key,
- defaultWarehousePath)
- sparkContext.conf.set(SQLConf.WAREHOUSE_PATH.key, warehousePath)
- sparkContext.conf.set("hive.metastore.warehouse.dir", warehousePath)
- logInfo(s"Setting warehouse location to $warehousePath")
- }
-
/**
* State shared across sessions, including the [[SparkContext]], cached data, listener,
* and a catalog that interacts with external systems.
http://git-wip-us.apache.org/repos/asf/spark/blob/7bf9b120/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
index 31f28f2..46579ec 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
@@ -17,8 +17,6 @@
package org.apache.spark.sql.hive
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.analysis.Analyzer
import org.apache.spark.sql.execution.SparkPlanner
http://git-wip-us.apache.org/repos/asf/spark/blob/7bf9b120/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSharedState.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSharedState.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSharedState.scala
index 0ea5ce9..f0d9640 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSharedState.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSharedState.scala
@@ -18,8 +18,9 @@
package org.apache.spark.sql.hive
import org.apache.spark.SparkContext
-import org.apache.spark.sql.hive.client.{HiveClient, HiveClientImpl}
-import org.apache.spark.sql.internal.SharedState
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.hive.client.HiveClient
+import org.apache.spark.sql.internal.{SharedState, SQLConf}
/**
@@ -27,9 +28,17 @@ import org.apache.spark.sql.internal.SharedState
* [[org.apache.spark.sql.SparkSession]] backed by Hive.
*/
private[hive] class HiveSharedState(override val sparkContext: SparkContext)
- extends SharedState(sparkContext) {
+ extends SharedState(sparkContext) with Logging {
- // TODO: just share the IsolatedClientLoader instead of the client instances themselves
+ // TODO: just share the IsolatedClientLoader instead of the client instance itself
+
+ {
+ // Set the Hive metastore warehouse path to the one we use
+ val tempConf = new SQLConf
+ sparkContext.conf.getAll.foreach { case (k, v) => tempConf.setConfString(k, v) }
+ sparkContext.conf.set("hive.metastore.warehouse.dir", tempConf.warehousePath)
+ logInfo(s"Setting Hive metastore warehouse path to '${tempConf.warehousePath}'")
+ }
/**
* A Hive client used to interact with the metastore.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org