You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2021/08/06 02:02:19 UTC
[spark] branch master updated: [SPARK-36421][SQL][DOCS] Use
ConfigEntry.key to fix docs and set command results
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c7fa3c9 [SPARK-36421][SQL][DOCS] Use ConfigEntry.key to fix docs and set command results
c7fa3c9 is described below
commit c7fa3c9090d88115c088570627138e1171c7a0eb
Author: Kent Yao <ya...@apache.org>
AuthorDate: Fri Aug 6 11:01:47 2021 +0900
[SPARK-36421][SQL][DOCS] Use ConfigEntry.key to fix docs and set command results
### What changes were proposed in this pull request?
This PR fixes the issue that `ConfigEntry` to be introduced to the doc field directly without calling `.key`, which causes malformed documents on the web site and in the result of `SET -v`
1. https://spark.apache.org/docs/3.1.2/configuration.html#static-sql-configuration - spark.sql.hive.metastore.jars
2. set -v
### Why are the changes needed?
bugfix
### Does this PR introduce _any_ user-facing change?
no, but contains doc fix
### How was this patch tested?
new tests
Closes #33647 from yaooqinn/SPARK-36421.
Authored-by: Kent Yao <ya...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 2 +-
sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala | 6 +++---
.../scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala | 7 ++++++-
3 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index c1b67f0..f85f745 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1014,7 +1014,7 @@ object SQLConf {
"This includes both datasource and converted Hive tables. When partition management " +
"is enabled, datasource tables store partition in the Hive metastore, and use the " +
s"metastore to prune partitions during query planning when " +
- s"$HIVE_METASTORE_PARTITION_PRUNING is set to true.")
+ s"${HIVE_METASTORE_PARTITION_PRUNING.key} is set to true.")
.version("2.1.1")
.booleanConf
.createWithDefault(true)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
index 3b010d9..93a38e5 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
@@ -92,9 +92,9 @@ private[spark] object HiveUtils extends Logging {
| 3. "path"
| Use Hive jars configured by `spark.sql.hive.metastore.jars.path`
| in comma separated format. Support both local or remote paths.The provided jars
- | should be the same version as ${HIVE_METASTORE_VERSION}.
+ | should be the same version as `${HIVE_METASTORE_VERSION.key}`.
| 4. A classpath in the standard format for both Hive and Hadoop. The provided jars
- | should be the same version as ${HIVE_METASTORE_VERSION}.
+ | should be the same version as `${HIVE_METASTORE_VERSION.key}`.
""".stripMargin)
.version("1.4.0")
.stringConf
@@ -103,7 +103,7 @@ private[spark] object HiveUtils extends Logging {
val HIVE_METASTORE_JARS_PATH = buildStaticConf("spark.sql.hive.metastore.jars.path")
.doc(s"""
| Comma-separated paths of the jars that used to instantiate the HiveMetastoreClient.
- | This configuration is useful only when `{$HIVE_METASTORE_JARS.key}` is set as `path`.
+ | This configuration is useful only when `${HIVE_METASTORE_JARS.key}` is set as `path`.
| The paths can be any of the following format:
| 1. file://path/to/jar/foo.jar
| 2. hdfs://nameservice/path/to/jar/foo.jar
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index a84db58..cc92a28 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -2635,7 +2635,12 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
}
@SlowHiveTest
-class SQLQuerySuite extends SQLQuerySuiteBase with DisableAdaptiveExecutionSuite
+class SQLQuerySuite extends SQLQuerySuiteBase with DisableAdaptiveExecutionSuite {
+ test("SPARK-36421: Validate all SQL configs to prevent from wrong use for ConfigEntry") {
+ val df = spark.sql("set -v").select("Meaning")
+ assert(df.collect().forall(!_.getString(0).contains("ConfigEntry")))
+ }
+}
@SlowHiveTest
class SQLQuerySuiteAE extends SQLQuerySuiteBase with EnableAdaptiveExecutionSuite
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org