You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/12/05 19:27:48 UTC
spark git commit: [SPARK-4761][SQL] Enables Kryo by default in Spark
SQL Thrift server
Repository: spark
Updated Branches:
refs/heads/master f5801e813 -> 6f61e1f96
[SPARK-4761][SQL] Enables Kryo by default in Spark SQL Thrift server
Enables Kryo and disables reference tracking by default in Spark SQL Thrift server. Configurations explicitly defined by users in `spark-defaults.conf` are respected (the Thrift server is started by `spark-submit`, which handles configuration properties properly).
<!-- Reviewable:start -->
[<img src="https://reviewable.io/review_button.png" height=40 alt="Review on Reviewable"/>](https://reviewable.io/reviews/apache/spark/3621)
<!-- Reviewable:end -->
Author: Cheng Lian <li...@databricks.com>
Closes #3621 from liancheng/kryo-by-default and squashes the following commits:
70c2775 [Cheng Lian] Enables Kryo by default in Spark SQL Thrift server
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6f61e1f9
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6f61e1f9
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6f61e1f9
Branch: refs/heads/master
Commit: 6f61e1f961826a6c9e98a66d10b271b7e3c7dd55
Parents: f5801e8
Author: Cheng Lian <li...@databricks.com>
Authored: Fri Dec 5 10:27:40 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Fri Dec 5 10:27:40 2014 -0800
----------------------------------------------------------------------
.../spark/sql/hive/thriftserver/SparkSQLEnv.scala | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6f61e1f9/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
----------------------------------------------------------------------
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
index 89732c9..158c225 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
@@ -32,11 +32,21 @@ private[hive] object SparkSQLEnv extends Logging {
def init() {
if (hiveContext == null) {
- val sparkConf = new SparkConf()
+ val sparkConf = new SparkConf(loadDefaults = true)
+ val maybeSerializer = sparkConf.getOption("spark.serializer")
+ val maybeKryoReferenceTracking = sparkConf.getOption("spark.kryo.referenceTracking")
+
+ sparkConf
.setAppName(s"SparkSQL::${java.net.InetAddress.getLocalHost.getHostName}")
.set("spark.sql.hive.version", HiveShim.version)
- sparkContext = new SparkContext(sparkConf)
+ .set(
+ "spark.serializer",
+ maybeSerializer.getOrElse("org.apache.spark.serializer.KryoSerializer"))
+ .set(
+ "spark.kryo.referenceTracking",
+ maybeKryoReferenceTracking.getOrElse("false"))
+ sparkContext = new SparkContext(sparkConf)
sparkContext.addSparkListener(new StatsReportListener())
hiveContext = new HiveContext(sparkContext)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org