You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/07/30 03:20:55 UTC
git commit: [SPARK-2631][SQL] Use SQLConf to configure in-memory
columnar caching
Repository: spark
Updated Branches:
refs/heads/master 39b819310 -> 86534d0f5
[SPARK-2631][SQL] Use SQLConf to configure in-memory columnar caching
Author: Michael Armbrust <mi...@databricks.com>
Closes #1638 from marmbrus/cachedConfig and squashes the following commits:
2362082 [Michael Armbrust] Use SQLConf to configure in-memory columnar caching
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/86534d0f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/86534d0f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/86534d0f
Branch: refs/heads/master
Commit: 86534d0f5255362618c05a07b0171ec35c915822
Parents: 39b8193
Author: Michael Armbrust <mi...@databricks.com>
Authored: Tue Jul 29 18:20:51 2014 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Tue Jul 29 18:20:51 2014 -0700
----------------------------------------------------------------------
sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala | 4 ++++
sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala | 2 --
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/86534d0f/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala
index be8d4e1..5d85a0f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala
@@ -22,6 +22,7 @@ import java.util.Properties
import scala.collection.JavaConverters._
object SQLConf {
+ val COMPRESS_CACHED = "spark.sql.inMemoryColumnarStorage.compressed"
val AUTO_BROADCASTJOIN_THRESHOLD = "spark.sql.autoBroadcastJoinThreshold"
val SHUFFLE_PARTITIONS = "spark.sql.shuffle.partitions"
val DEFAULT_SIZE_IN_BYTES = "spark.sql.defaultSizeInBytes"
@@ -49,6 +50,9 @@ trait SQLConf {
/** ************************ Spark SQL Params/Hints ******************* */
// TODO: refactor so that these hints accessors don't pollute the name space of SQLContext?
+ /** When true tables cached using the in-memory columnar caching will be compressed. */
+ private[spark] def useCompression: Boolean = get(COMPRESS_CACHED, "false").toBoolean
+
/** Number of partitions to use for shuffle operators. */
private[spark] def numShufflePartitions: Int = get(SHUFFLE_PARTITIONS, "200").toInt
http://git-wip-us.apache.org/repos/asf/spark/blob/86534d0f/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index a136c7b..c2bdef7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -192,8 +192,6 @@ class SQLContext(@transient val sparkContext: SparkContext)
currentTable.logicalPlan
case _ =>
- val useCompression =
- sparkContext.conf.getBoolean("spark.sql.inMemoryColumnarStorage.compressed", false)
InMemoryRelation(useCompression, executePlan(currentTable).executedPlan)
}