You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2018/01/29 12:09:12 UTC
spark git commit: [SPARK-23238][SQL] Externalize SQLConf
configurations exposed in documentation
Repository: spark
Updated Branches:
refs/heads/master 49b0207dc -> 39d2c6b03
[SPARK-23238][SQL] Externalize SQLConf configurations exposed in documentation
## What changes were proposed in this pull request?
This PR proposes to expose few internal configurations found in the documentation.
Also it fixes the description for `spark.sql.execution.arrow.enabled`.
It's quite self-explanatory.
## How was this patch tested?
N/A
Author: hyukjinkwon <gu...@gmail.com>
Closes #20403 from HyukjinKwon/minor-doc-arrow.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/39d2c6b0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/39d2c6b0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/39d2c6b0
Branch: refs/heads/master
Commit: 39d2c6b03488895a0acb1dd3c46329db00fdd357
Parents: 49b0207
Author: hyukjinkwon <gu...@gmail.com>
Authored: Mon Jan 29 21:09:05 2018 +0900
Committer: hyukjinkwon <gu...@gmail.com>
Committed: Mon Jan 29 21:09:05 2018 +0900
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/internal/SQLConf.scala | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/39d2c6b0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 2c70b00..61ea03d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -123,14 +123,12 @@ object SQLConf {
.createWithDefault(10)
val COMPRESS_CACHED = buildConf("spark.sql.inMemoryColumnarStorage.compressed")
- .internal()
.doc("When set to true Spark SQL will automatically select a compression codec for each " +
"column based on statistics of the data.")
.booleanConf
.createWithDefault(true)
val COLUMN_BATCH_SIZE = buildConf("spark.sql.inMemoryColumnarStorage.batchSize")
- .internal()
.doc("Controls the size of batches for columnar caching. Larger batch sizes can improve " +
"memory utilization and compression, but risk OOMs when caching data.")
.intConf
@@ -1043,11 +1041,11 @@ object SQLConf {
val ARROW_EXECUTION_ENABLE =
buildConf("spark.sql.execution.arrow.enabled")
- .internal()
- .doc("Make use of Apache Arrow for columnar data transfers. Currently available " +
- "for use with pyspark.sql.DataFrame.toPandas with the following data types: " +
- "StringType, BinaryType, BooleanType, DoubleType, FloatType, ByteType, IntegerType, " +
- "LongType, ShortType")
+ .doc("When true, make use of Apache Arrow for columnar data transfers. Currently available " +
+ "for use with pyspark.sql.DataFrame.toPandas, and " +
+ "pyspark.sql.SparkSession.createDataFrame when its input is a Pandas DataFrame. " +
+ "The following data types are unsupported: " +
+ "MapType, ArrayType of TimestampType, and nested StructType.")
.booleanConf
.createWithDefault(false)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org