You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2018/09/27 04:38:33 UTC
spark git commit: Revert "[SPARK-25540][SQL][PYSPARK] Make
HiveContext in PySpark behave as the same as Scala."
Repository: spark
Updated Branches:
refs/heads/branch-2.4 f12769e73 -> 01c000b52
Revert "[SPARK-25540][SQL][PYSPARK] Make HiveContext in PySpark behave as the same as Scala."
This reverts commit 7656358adc39eb8eb881368ab5a066fbf86149c8.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/01c000b5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/01c000b5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/01c000b5
Branch: refs/heads/branch-2.4
Commit: 01c000b522d4b2e0ebbe9617be591e5de3b59dea
Parents: f12769e
Author: hyukjinkwon <gu...@apache.org>
Authored: Thu Sep 27 12:38:14 2018 +0800
Committer: hyukjinkwon <gu...@apache.org>
Committed: Thu Sep 27 12:38:14 2018 +0800
----------------------------------------------------------------------
python/pyspark/sql/context.py | 3 +--
python/pyspark/sql/session.py | 19 +++++--------------
2 files changed, 6 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/01c000b5/python/pyspark/sql/context.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 1938965..9c094dd 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -485,8 +485,7 @@ class HiveContext(SQLContext):
"SparkSession.builder.enableHiveSupport().getOrCreate() instead.",
DeprecationWarning)
if jhiveContext is None:
- sparkContext._conf.set("spark.sql.catalogImplementation", "hive")
- sparkSession = SparkSession.builder._sparkContext(sparkContext).getOrCreate()
+ sparkSession = SparkSession.builder.enableHiveSupport().getOrCreate()
else:
sparkSession = SparkSession(sparkContext, jhiveContext.sparkSession())
SQLContext.__init__(self, sparkContext, sparkSession, jhiveContext)
http://git-wip-us.apache.org/repos/asf/spark/blob/01c000b5/python/pyspark/sql/session.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py
index a5e2872..51a38eb 100644
--- a/python/pyspark/sql/session.py
+++ b/python/pyspark/sql/session.py
@@ -83,7 +83,6 @@ class SparkSession(object):
_lock = RLock()
_options = {}
- _sc = None
@since(2.0)
def config(self, key=None, value=None, conf=None):
@@ -140,11 +139,6 @@ class SparkSession(object):
"""
return self.config("spark.sql.catalogImplementation", "hive")
- def _sparkContext(self, sc):
- with self._lock:
- self._sc = sc
- return self
-
@since(2.0)
def getOrCreate(self):
"""Gets an existing :class:`SparkSession` or, if there is no existing one, creates a
@@ -173,14 +167,11 @@ class SparkSession(object):
from pyspark.conf import SparkConf
session = SparkSession._instantiatedSession
if session is None or session._sc._jsc is None:
- if self._sc is not None:
- sc = self._sc
- else:
- sparkConf = SparkConf()
- for key, value in self._options.items():
- sparkConf.set(key, value)
- sc = SparkContext.getOrCreate(sparkConf)
- # This SparkContext may be an existing one.
+ sparkConf = SparkConf()
+ for key, value in self._options.items():
+ sparkConf.set(key, value)
+ sc = SparkContext.getOrCreate(sparkConf)
+ # This SparkContext may be an existing one.
for key, value in self._options.items():
# we need to propagate the confs
# before we create the SparkSession. Otherwise, confs like
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org