You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by hv...@apache.org on 2025/02/06 13:35:05 UTC
(spark) branch branch-4.0 updated: [SPARK-49371][SPARK-51086][SQL][CONNECT][DOCS] Fix various doc SQL issues
This is an automated email from the ASF dual-hosted git repository.
hvanhovell pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 012f5be0f5e9 [SPARK-49371][SPARK-51086][SQL][CONNECT][DOCS] Fix various doc SQL issues
012f5be0f5e9 is described below
commit 012f5be0f5e9e7567a4acd6ccb22d2e6fa3e0d8b
Author: Herman van Hovell <he...@databricks.com>
AuthorDate: Thu Feb 6 09:34:22 2025 -0400
[SPARK-49371][SPARK-51086][SQL][CONNECT][DOCS] Fix various doc SQL issues
### What changes were proposed in this pull request?
This PR fixes a bunch of issues with SQL API docs for Scala and Java:
- Removes the now defunct Connect API docs. The current SQL docs provide enough coverage.
- Removes the following internal packages from the docs: `org.apache.spark.sql.artifact`, `org.apache.spark.sql.scripting`, and `org.apache.spark.sql.ml`.
- Fixes the removal of the `org.apache.spark.error` and `org.apache.spark.sql.error` packages.
- Marks a bunch of internal classes in the `org.apache.spark.sql` as `private[sql]` to remove them from the docs.
- Moves the `TableValuedFunctionArgument` interface from `org.apache.spark.sql` to `org.apache.spark.sql.internal` because it kept showing up in the docs.
### Why are the changes needed?
Readable docs are important!
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Manual inspection of the Java and Scala docs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #49800 from hvanhovell/SPARK-49371.
Authored-by: Herman van Hovell <he...@databricks.com>
Signed-off-by: Herman van Hovell <he...@databricks.com>
(cherry picked from commit aefaa6694567ce4cb3beda27e0916a8f47cf380d)
Signed-off-by: Herman van Hovell <he...@databricks.com>
---
docs/_plugins/build_api_docs.rb | 2 -
docs/index.md | 1 -
project/SparkBuild.scala | 61 ++++------------------
.../main/scala/org/apache/spark/sql/Column.scala | 2 +-
.../scala/org/apache/spark/sql/SQLContext.scala | 2 +-
.../scala/org/apache/spark/sql/SparkSession.scala | 6 +--
.../TableValuedFunctionArgument.scala | 4 +-
.../main/scala/org/apache/spark/sql/TableArg.scala | 3 +-
.../python/UserDefinedPythonFunction.scala | 3 +-
9 files changed, 21 insertions(+), 63 deletions(-)
diff --git a/docs/_plugins/build_api_docs.rb b/docs/_plugins/build_api_docs.rb
index eca0e5814584..0aa0db0bf989 100644
--- a/docs/_plugins/build_api_docs.rb
+++ b/docs/_plugins/build_api_docs.rb
@@ -149,11 +149,9 @@ def build_scala_and_java_docs
# Copy over the unified ScalaDoc for all projects to api/scala.
# This directory will be copied over to _site when `jekyll` command is run.
copy_and_update_scala_docs("../target/scala-2.13/unidoc", "api/scala")
- # copy_and_update_scala_docs("../sql/connect/client/jvm/target/scala-2.13/unidoc", "api/connect/scala")
# Copy over the unified JavaDoc for all projects to api/java.
copy_and_update_java_docs("../target/javaunidoc", "api/java", "api/scala")
- # copy_and_update_java_docs("../sql/connect/client/jvm/target/javaunidoc", "api/connect/java", "api/connect/scala")
end
def build_python_docs
diff --git a/docs/index.md b/docs/index.md
index fea62865e216..88ea8ad9efc3 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -117,7 +117,6 @@ options for deployment:
* [Spark Python API (Sphinx)](api/python/index.html)
* [Spark Scala API (Scaladoc)](api/scala/org/apache/spark/index.html)
-* [Spark Connect Scala Client API (Scaladoc)](api/connect/scala/org/apache/spark/index.html)
* [Spark Java API (Javadoc)](api/java/index.html)
* [Spark R API (Roxygen2)](api/R/index.html)
* [Spark SQL, Built-in Functions (MkDocs)](api/sql/index.html)
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 1bb74878c607..c16b000a4d3a 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -401,10 +401,7 @@ object SparkBuild extends PomBuild {
enable(PySparkAssembly.settings)(assembly)
/* Enable unidoc only for the root spark project */
- enable(SparkUnidoc.settings)(spark)
-
- /* Enable unidoc only for the root spark connect client project */
- // enable(SparkConnectClientUnidoc.settings)(connectClient)
+ enable(Unidoc.settings)(spark)
/* Sql-api ANTLR generation settings */
enable(SqlApi.settings)(sqlApi)
@@ -1333,7 +1330,7 @@ object SparkR {
)
}
-trait SharedUnidocSettings {
+object Unidoc {
import BuildCommons._
import sbtunidoc.BaseUnidocPlugin
@@ -1366,14 +1363,18 @@ trait SharedUnidocSettings {
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/util/collection")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/util/io")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/util/kvstore")))
+ .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/artifact")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/catalyst")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/connect/")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/classic/")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/execution")))
+ .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/internal")))
+ .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/scripting")))
+ .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/ml")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/hive")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/catalog/v2/utils")))
- .map(_.filterNot(_.getCanonicalPath.contains("org.apache.spark.errors")))
- .map(_.filterNot(_.getCanonicalPath.contains("org.apache.spark.sql.errors")))
+ .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/errors")))
+ .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/errors")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/hive")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/v2/avro")))
.map(_.filterNot(_.getCanonicalPath.contains("SSLOptions")))
@@ -1389,7 +1390,7 @@ trait SharedUnidocSettings {
val unidocSourceBase = settingKey[String]("Base URL of source links in Scaladoc.")
- lazy val baseSettings = BaseUnidocPlugin.projectSettings ++
+ lazy val settings = BaseUnidocPlugin.projectSettings ++
ScalaUnidocPlugin.projectSettings ++
JavaUnidocPlugin.projectSettings ++
Seq (
@@ -1446,27 +1447,7 @@ trait SharedUnidocSettings {
} else {
Seq()
}
- )
- )
-}
-
-object SparkUnidoc extends SharedUnidocSettings {
-
- import BuildCommons._
- import sbtunidoc.BaseUnidocPlugin
- import sbtunidoc.JavaUnidocPlugin
- import sbtunidoc.ScalaUnidocPlugin
- import sbtunidoc.BaseUnidocPlugin.autoImport._
- import sbtunidoc.GenJavadocPlugin.autoImport._
- import sbtunidoc.JavaUnidocPlugin.autoImport._
- import sbtunidoc.ScalaUnidocPlugin.autoImport._
-
- override def ignoreUndocumentedPackages(packages: Seq[Seq[File]]): Seq[Seq[File]] = {
- super.ignoreUndocumentedPackages(packages)
- .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/internal")))
- }
-
- lazy val settings = baseSettings ++ Seq(
+ ),
(ScalaUnidoc / unidoc / unidocProjectFilter) :=
inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, kubernetes,
yarn, tags, streamingKafka010, sqlKafka010, connectCommon, connect, connectClient,
@@ -1478,28 +1459,6 @@ object SparkUnidoc extends SharedUnidocSettings {
)
}
-object SparkConnectClientUnidoc extends SharedUnidocSettings {
-
- import BuildCommons._
- import sbtunidoc.BaseUnidocPlugin
- import sbtunidoc.JavaUnidocPlugin
- import sbtunidoc.ScalaUnidocPlugin
- import sbtunidoc.BaseUnidocPlugin.autoImport._
- import sbtunidoc.GenJavadocPlugin.autoImport._
- import sbtunidoc.JavaUnidocPlugin.autoImport._
- import sbtunidoc.ScalaUnidocPlugin.autoImport._
-
- override def ignoreUndocumentedPackages(packages: Seq[Seq[File]]): Seq[Seq[File]] = {
- super.ignoreUndocumentedPackages(packages)
- .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/application")))
- }
-
- lazy val settings = baseSettings ++ Seq(
- (ScalaUnidoc / unidoc / unidocProjectFilter) := inProjects(connectClient, connectCommon, sqlApi),
- (JavaUnidoc / unidoc / unidocProjectFilter) := inProjects(connectClient, connectCommon, sqlApi),
- )
-}
-
object Checkstyle {
lazy val settings = Seq(
checkstyleSeverityLevel := CheckstyleSeverityLevel.Error,
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/Column.scala b/sql/api/src/main/scala/org/apache/spark/sql/Column.scala
index f13b340e5e9c..7f5eed1eb1ad 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/Column.scala
@@ -26,7 +26,7 @@ import org.apache.spark.sql.catalyst.parser.DataTypeParser
import org.apache.spark.sql.catalyst.trees.CurrentOrigin.withOrigin
import org.apache.spark.sql.expressions.Window
import org.apache.spark.sql.functions.{lit, map}
-import org.apache.spark.sql.internal.ColumnNode
+import org.apache.spark.sql.internal.{ColumnNode, TableValuedFunctionArgument}
import org.apache.spark.sql.types._
import org.apache.spark.util.ArrayImplicits._
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/api/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 8da22791f603..848a0215240b 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -978,7 +978,7 @@ abstract class SQLContext private[sql] (val sparkSession: SparkSession)
* scenario, setActive could set a SQLContext for current thread, which will be returned by
* getOrCreate instead of the global one.
*/
-trait SQLContextCompanion {
+private[sql] trait SQLContextCompanion {
private[sql] type SQLContextImpl <: SQLContext
private[sql] type SparkContextImpl <: SparkContext
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/api/src/main/scala/org/apache/spark/sql/SparkSession.scala
index e94ecb4f26fc..86073bb4f869 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -1065,7 +1065,7 @@ private[sql] abstract class SparkSessionCompanion {
/**
* This object keeps track of the global (default) and the thread-local SparkSession.
*/
-object SparkSessionCompanion {
+private[sql] object SparkSessionCompanion {
/** The active SparkSession for the current thread. */
private val activeThreadSession = new InheritableThreadLocal[SparkSession]
@@ -1078,7 +1078,7 @@ object SparkSessionCompanion {
* Builder for [[SparkSession]].
*/
@Stable
-abstract class SparkSessionBuilder {
+private[sql] abstract class SparkSessionBuilder {
import SparkSessionBuilder._
protected val options = new scala.collection.mutable.HashMap[String, String]
@@ -1248,7 +1248,7 @@ abstract class SparkSessionBuilder {
def create(): SparkSession
}
-object SparkSessionBuilder {
+private[sql] object SparkSessionBuilder {
val MASTER_KEY = "spark.master"
val APP_NAME_KEY = "spark.app.name"
val CATALOG_IMPL_KEY = "spark.sql.catalogImplementation"
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/TableValuedFunctionArgument.scala b/sql/api/src/main/scala/org/apache/spark/sql/internal/TableValuedFunctionArgument.scala
similarity index 90%
rename from sql/api/src/main/scala/org/apache/spark/sql/TableValuedFunctionArgument.scala
rename to sql/api/src/main/scala/org/apache/spark/sql/internal/TableValuedFunctionArgument.scala
index f99c4ecd4855..10674b6b389c 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/TableValuedFunctionArgument.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/internal/TableValuedFunctionArgument.scala
@@ -15,6 +15,6 @@
* limitations under the License.
*/
-package org.apache.spark.sql
+package org.apache.spark.sql.internal
-trait TableValuedFunctionArgument
+private[sql] trait TableValuedFunctionArgument
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/TableArg.scala b/sql/core/src/main/scala/org/apache/spark/sql/TableArg.scala
index bdacdcca24fd..01da9a8d1d7f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/TableArg.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/TableArg.scala
@@ -18,8 +18,9 @@
package org.apache.spark.sql
import org.apache.spark.sql.catalyst.expressions.{Ascending, Expression, FunctionTableSubqueryArgumentExpression, SortOrder}
+import org.apache.spark.sql.internal.TableValuedFunctionArgument
-class TableArg(
+private[sql] class TableArg(
private[sql] val expression: FunctionTableSubqueryArgumentExpression,
sparkSession: classic.SparkSession)
extends TableValuedFunctionArgument {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonFunction.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonFunction.scala
index c6cb12b1a59f..5b508d1a0a72 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonFunction.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonFunction.scala
@@ -24,7 +24,7 @@ import scala.collection.mutable.ArrayBuffer
import net.razorvine.pickle.Pickler
import org.apache.spark.api.python.{PythonEvalType, PythonFunction, PythonWorkerUtils, SpecialLengths}
-import org.apache.spark.sql.{Column, TableArg, TableValuedFunctionArgument}
+import org.apache.spark.sql.{Column, TableArg}
import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, Descending, Expression, FunctionTableSubqueryArgumentExpression, NamedArgumentExpression, NullsFirst, NullsLast, PythonUDAF, PythonUDF, PythonUDTF, PythonUDTFAnalyzeResult, PythonUDTFSelectedExpression, SortOrder, UnresolvedPolymorphicPythonUDTF}
import org.apache.spark.sql.catalyst.parser.ParserInterface
import org.apache.spark.sql.catalyst.plans.logical.{Generate, LogicalPlan, NamedParametersSupport, OneRowRelation}
@@ -32,6 +32,7 @@ import org.apache.spark.sql.classic.{DataFrame, Dataset, SparkSession}
import org.apache.spark.sql.classic.ClassicConversions._
import org.apache.spark.sql.classic.ExpressionUtils.expression
import org.apache.spark.sql.errors.QueryCompilationErrors
+import org.apache.spark.sql.internal.TableValuedFunctionArgument
import org.apache.spark.sql.types.{DataType, StructType}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org