You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ge...@apache.org on 2021/08/25 14:33:44 UTC
[spark] branch branch-3.2 updated: [SPARK-36585][SQL][DOCS] Support
setting "since" version in FunctionRegistry
This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 4648412 [SPARK-36585][SQL][DOCS] Support setting "since" version in FunctionRegistry
4648412 is described below
commit 464841224ccf14b8fd8a1358e4e6b219f36fd1be
Author: Gengliang Wang <ge...@apache.org>
AuthorDate: Wed Aug 25 22:32:20 2021 +0800
[SPARK-36585][SQL][DOCS] Support setting "since" version in FunctionRegistry
### What changes were proposed in this pull request?
Spark 3.2.0 includes two new functions `regexp` and `regexp_like`, which are identical to `rlike`. However, in the generated documentation. the since versions of both functions are `1.0.0` since they are based on the expression `RLike`:
- https://dist.apache.org/repos/dist/dev/spark/v3.2.0-rc1-docs/_site/api/sql/index.html#regexp
- https://dist.apache.org/repos/dist/dev/spark/v3.2.0-rc1-docs/_site/api/sql/index.html#regexp_like
This PR is to:
* Support setting `since` version in FunctionRegistry
* Correct the `since` version of `regexp` and `regexp_like`
### Why are the changes needed?
Correct the SQL doc
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Run
```
sh sql/create-docs.sh
```
and check the SQL doc manually
Closes #33834 from gengliangwang/allowSQLFunVersion.
Authored-by: Gengliang Wang <ge...@apache.org>
Signed-off-by: Gengliang Wang <ge...@apache.org>
(cherry picked from commit 18143fb42604a3ad7ed4bfd89084f7f0a6bce4d6)
Signed-off-by: Gengliang Wang <ge...@apache.org>
---
.../sql/catalyst/analysis/FunctionRegistry.scala | 33 ++++++++++++++--------
1 file changed, 22 insertions(+), 11 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 63b1525..00ece4d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -107,7 +107,9 @@ object FunctionRegistryBase {
* Return an expression info and a function builder for the function as defined by
* T using the given name.
*/
- def build[T : ClassTag](name: String): (ExpressionInfo, Seq[Expression] => T) = {
+ def build[T : ClassTag](
+ name: String,
+ since: Option[String]): (ExpressionInfo, Seq[Expression] => T) = {
val runtimeClass = scala.reflect.classTag[T].runtimeClass
// For `RuntimeReplaceable`, skip the constructor with most arguments, which is the main
// constructor and contains non-parameter `child` and should not be used as function builder.
@@ -150,13 +152,13 @@ object FunctionRegistryBase {
}
}
- (expressionInfo(name), builder)
+ (expressionInfo(name, since), builder)
}
/**
* Creates an [[ExpressionInfo]] for the function as defined by T using the given name.
*/
- def expressionInfo[T : ClassTag](name: String): ExpressionInfo = {
+ def expressionInfo[T : ClassTag](name: String, since: Option[String]): ExpressionInfo = {
val clazz = scala.reflect.classTag[T].runtimeClass
val df = clazz.getAnnotation(classOf[ExpressionDescription])
if (df != null) {
@@ -170,7 +172,7 @@ object FunctionRegistryBase {
df.examples(),
df.note(),
df.group(),
- df.since(),
+ since.getOrElse(df.since()),
df.deprecated(),
df.source())
} else {
@@ -495,12 +497,12 @@ object FunctionRegistry {
expression[RegExpExtract]("regexp_extract"),
expression[RegExpExtractAll]("regexp_extract_all"),
expression[RegExpReplace]("regexp_replace"),
- expression[RLike]("regexp_like", true),
- expression[RLike]("regexp", true),
expression[StringRepeat]("repeat"),
expression[StringReplace]("replace"),
expression[Overlay]("overlay"),
expression[RLike]("rlike"),
+ expression[RLike]("regexp_like", true, Some("3.2.0")),
+ expression[RLike]("regexp", true, Some("3.2.0")),
expression[StringRPad]("rpad"),
expression[StringTrimRight]("rtrim"),
expression[Sentences]("sentences"),
@@ -735,10 +737,19 @@ object FunctionRegistry {
val functionSet: Set[FunctionIdentifier] = builtin.listFunction().toSet
- /** See usage above. */
- private def expression[T <: Expression : ClassTag](name: String, setAlias: Boolean = false)
- : (String, (ExpressionInfo, FunctionBuilder)) = {
- val (expressionInfo, builder) = FunctionRegistryBase.build[T](name)
+ /**
+ * Create a SQL function builder and corresponding `ExpressionInfo`.
+ * @param name The function name.
+ * @param setAlias The alias name used in SQL representation string.
+ * @param since The Spark version since the function is added.
+ * @tparam T The actual expression class.
+ * @return (function name, (expression information, function builder))
+ */
+ private def expression[T <: Expression : ClassTag](
+ name: String,
+ setAlias: Boolean = false,
+ since: Option[String] = None): (String, (ExpressionInfo, FunctionBuilder)) = {
+ val (expressionInfo, builder) = FunctionRegistryBase.build[T](name, since)
val newBuilder = (expressions: Seq[Expression]) => {
val expr = builder(expressions)
if (setAlias) expr.setTagValue(FUNC_ALIAS, name)
@@ -813,7 +824,7 @@ object TableFunctionRegistry {
private def logicalPlan[T <: LogicalPlan : ClassTag](name: String)
: (String, (ExpressionInfo, TableFunctionBuilder)) = {
- val (info, builder) = FunctionRegistryBase.build[T](name)
+ val (info, builder) = FunctionRegistryBase.build[T](name, since = None)
val newBuilder = (expressions: Seq[Expression]) => {
try {
builder(expressions)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org