You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2020/04/29 06:37:38 UTC
[spark] branch branch-3.0 updated: [SPARK-31596][SQL][DOCS]
Generate SQL Configurations from hive module to configuration doc
This is an automated email from the ASF dual-hosted git repository.
yamamuro pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new cf6a3de [SPARK-31596][SQL][DOCS] Generate SQL Configurations from hive module to configuration doc
cf6a3de is described below
commit cf6a3de217d4fae7d46ab024b0a13ccdc9cde907
Author: Kent Yao <ya...@hotmail.com>
AuthorDate: Wed Apr 29 15:34:45 2020 +0900
[SPARK-31596][SQL][DOCS] Generate SQL Configurations from hive module to configuration doc
### What changes were proposed in this pull request?
This PR adds `-Phive` profile to the pre-build phase to build the hive module to dev classpath.
Then reflect the HiveUtils object to dump all configurations in the class.
### Why are the changes needed?
supply SQL configurations from hive module to doc
### Does this PR introduce any user-facing change?
NO
### How was this patch tested?
passing Jenkins
add verified locally
Closes #28394 from yaooqinn/SPARK-31596.
Authored-by: Kent Yao <ya...@hotmail.com>
Signed-off-by: Takeshi Yamamuro <ya...@apache.org>
(cherry picked from commit 295d866969e01e70835210880ed27022d837a8f4)
Signed-off-by: Takeshi Yamamuro <ya...@apache.org>
---
docs/_plugins/copy_api_dirs.rb | 4 ++--
.../org/apache/spark/sql/api/python/PythonSQLUtils.scala | 14 +++++++++++++-
2 files changed, 15 insertions(+), 3 deletions(-)
diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb
index f95e4e2..8e2a06e 100644
--- a/docs/_plugins/copy_api_dirs.rb
+++ b/docs/_plugins/copy_api_dirs.rb
@@ -157,8 +157,8 @@ if not (ENV['SKIP_API'] == '1')
curr_dir = pwd
cd("..")
- puts "Running 'build/sbt clean package' from " + pwd + "; this may take a few minutes..."
- system("build/sbt clean package") || raise("SQL doc generation failed")
+ puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this may take a few minutes..."
+ system("build/sbt clean package -Phive") || raise("SQL doc generation failed")
puts "Moving back into docs dir."
cd("docs")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
index 89bb4f3..3825460 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
@@ -20,10 +20,14 @@ package org.apache.spark.sql.api.python
import java.io.InputStream
import java.nio.channels.Channels
+import scala.util.control.NonFatal
+
import org.apache.spark.api.java.JavaRDD
import org.apache.spark.api.python.PythonRDDServer
+import org.apache.spark.internal.Logging
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, SQLContext}
+import org.apache.spark.sql.catalyst.ScalaReflection
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
@@ -32,7 +36,7 @@ import org.apache.spark.sql.execution.arrow.ArrowConverters
import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
import org.apache.spark.sql.types.DataType
-private[sql] object PythonSQLUtils {
+private[sql] object PythonSQLUtils extends Logging {
def parseDataType(typeText: String): DataType = CatalystSqlParser.parseDataType(typeText)
// This is needed when generating SQL documentation for built-in functions.
@@ -44,6 +48,14 @@ private[sql] object PythonSQLUtils {
val conf = new SQLConf()
// Force to build static SQL configurations
StaticSQLConf
+ // Force to build SQL configurations from Hive module
+ try {
+ val symbol = ScalaReflection.mirror.staticModule("org.apache.spark.sql.hive.HiveUtils")
+ ScalaReflection.mirror.reflectModule(symbol).instance
+ } catch {
+ case NonFatal(e) =>
+ logWarning("Cannot generated sql configurations from hive module", e)
+ }
conf.getAllDefinedConfs
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org