You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2020/09/29 05:34:42 UTC
[spark] branch branch-3.0 updated: [MINOR][DOCS] Document when
`current_date` and `current_timestamp` are evaluated
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 118de10 [MINOR][DOCS] Document when `current_date` and `current_timestamp` are evaluated
118de10 is described below
commit 118de10b5963427e88820704a02af97b73b26942
Author: Max Gekk <ma...@gmail.com>
AuthorDate: Tue Sep 29 05:20:12 2020 +0000
[MINOR][DOCS] Document when `current_date` and `current_timestamp` are evaluated
### What changes were proposed in this pull request?
Explicitly document that `current_date` and `current_timestamp` are executed at the start of query evaluation. And all calls of `current_date`/`current_timestamp` within the same query return the same value
### Why are the changes needed?
Users could expect that `current_date` and `current_timestamp` return the current date/timestamp at the moment of query execution but in fact the functions are folded by the optimizer at the start of query evaluation:
https://github.com/apache/spark/blob/0df8dd60733066076967f0525210bbdb5e12415a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala#L71-L91
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
by running `./dev/scalastyle`.
Closes #29892 from MaxGekk/doc-current_date.
Authored-by: Max Gekk <ma...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit 1b60ff5afea0637f74c5f064642225b35b13b069)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
R/pkg/R/functions.R | 6 ++++--
python/pyspark/sql/functions.py | 6 ++++--
.../spark/sql/catalyst/expressions/datetimeExpressions.scala | 12 ++++++------
sql/core/src/main/scala/org/apache/spark/sql/functions.scala | 6 ++++--
4 files changed, 18 insertions(+), 12 deletions(-)
diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R
index edec2f4..4cb23d7 100644
--- a/R/pkg/R/functions.R
+++ b/R/pkg/R/functions.R
@@ -4028,7 +4028,8 @@ setMethod("date_trunc",
})
#' @details
-#' \code{current_date}: Returns the current date as a date column.
+#' \code{current_date}: Returns the current date at the start of query evaluation as a date column.
+#' All calls of current_date within the same query return the same value.
#'
#' @rdname column_datetime_functions
#' @aliases current_date current_date,missing-method
@@ -4044,7 +4045,8 @@ setMethod("current_date",
})
#' @details
-#' \code{current_timestamp}: Returns the current timestamp as a timestamp column.
+#' \code{current_timestamp}: Returns the current timestamp at the start of query evaluation as
+#' a timestamp column. All calls of current_timestamp within the same query return the same value.
#'
#' @rdname column_datetime_functions
#' @aliases current_timestamp current_timestamp,missing-method
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 0bf6bc6..041f841 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -909,7 +909,8 @@ def ntile(n):
@since(1.5)
def current_date():
"""
- Returns the current date as a :class:`DateType` column.
+ Returns the current date at the start of query evaluation as a :class:`DateType` column.
+ All calls of current_date within the same query return the same value.
"""
sc = SparkContext._active_spark_context
return Column(sc._jvm.functions.current_date())
@@ -917,7 +918,8 @@ def current_date():
def current_timestamp():
"""
- Returns the current timestamp as a :class:`TimestampType` column.
+ Returns the current timestamp at the start of query evaluation as a :class:`TimestampType`
+ column. All calls of current_timestamp within the same query return the same value.
"""
sc = SparkContext._active_spark_context
return Column(sc._jvm.functions.current_timestamp())
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 62f38b3..b2ff2fa 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -58,13 +58,12 @@ trait TimeZoneAwareExpression extends Expression {
/**
* Returns the current date at the start of query evaluation.
- * All calls of current_date within the same query return the same value.
- *
* There is no code generation since this expression should get constant folded by the optimizer.
*/
+// scalastyle:off line.size.limit
@ExpressionDescription(
usage = """
- _FUNC_() - Returns the current date at the start of query evaluation.
+ _FUNC_() - Returns the current date at the start of query evaluation. All calls of current_date within the same query return the same value.
_FUNC_ - Returns the current date at the start of query evaluation.
""",
@@ -80,6 +79,7 @@ trait TimeZoneAwareExpression extends Expression {
""",
group = "datetime_funcs",
since = "1.5.0")
+// scalastyle:on line.size.limit
case class CurrentDate(timeZoneId: Option[String] = None)
extends LeafExpression with TimeZoneAwareExpression with CodegenFallback {
@@ -107,13 +107,12 @@ abstract class CurrentTimestampLike() extends LeafExpression with CodegenFallbac
/**
* Returns the current timestamp at the start of query evaluation.
- * All calls of current_timestamp within the same query return the same value.
- *
* There is no code generation since this expression should get constant folded by the optimizer.
*/
+// scalastyle:off line.size.limit
@ExpressionDescription(
usage = """
- _FUNC_() - Returns the current timestamp at the start of query evaluation.
+ _FUNC_() - Returns the current timestamp at the start of query evaluation. All calls of current_timestamp within the same query return the same value.
_FUNC_ - Returns the current timestamp at the start of query evaluation.
""",
@@ -129,6 +128,7 @@ abstract class CurrentTimestampLike() extends LeafExpression with CodegenFallbac
""",
group = "datetime_funcs",
since = "1.5.0")
+// scalastyle:on line.size.limit
case class CurrentTimestamp() extends CurrentTimestampLike {
override def prettyName: String = "current_timestamp"
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index cf889f5..ff8ee25 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -2692,7 +2692,8 @@ object functions {
}
/**
- * Returns the current date as a date column.
+ * Returns the current date at the start of query evaluation as a date column.
+ * All calls of current_date within the same query return the same value.
*
* @group datetime_funcs
* @since 1.5.0
@@ -2700,7 +2701,8 @@ object functions {
def current_date(): Column = withExpr { CurrentDate() }
/**
- * Returns the current timestamp as a timestamp column.
+ * Returns the current timestamp at the start of query evaluation as a timestamp column.
+ * All calls of current_timestamp within the same query return the same value.
*
* @group datetime_funcs
* @since 1.5.0
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org