You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2022/12/23 05:38:35 UTC
[spark] branch master updated: [SPARK-41647][CONNECT] Deduplicate docstrings in pyspark.sql.connect.functions
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5b1ff91ae05 [SPARK-41647][CONNECT] Deduplicate docstrings in pyspark.sql.connect.functions
5b1ff91ae05 is described below
commit 5b1ff91ae0559cb7ce47770043209d225528f65b
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Fri Dec 23 14:38:21 2022 +0900
[SPARK-41647][CONNECT] Deduplicate docstrings in pyspark.sql.connect.functions
### What changes were proposed in this pull request?
This PR proposes to deduplicate docstrings in `pyspark.sql.connect.functions`.
### Why are the changes needed?
For easier maintenance
### Does this PR introduce _any_ user-facing change?
No, dev-only. There're mintor doc changes that mention about remote support in Apache Spark 3.4.
### How was this patch tested?
No test. It has to be manually verified until we resolve SPARK-41653.
Closes #39181 from HyukjinKwon/SPARK-41647.
Authored-by: Hyukjin Kwon <gu...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/sql/connect/functions.py | 7049 +++----------------------------
python/pyspark/sql/functions.py | 744 +++-
2 files changed, 1260 insertions(+), 6533 deletions(-)
diff --git a/python/pyspark/sql/connect/functions.py b/python/pyspark/sql/connect/functions.py
index 78a3265cb38..c2b1c7d61c8 100644
--- a/python/pyspark/sql/connect/functions.py
+++ b/python/pyspark/sql/connect/functions.py
@@ -17,6 +17,7 @@
import inspect
import warnings
+from typing import Any, TYPE_CHECKING, Union, List, overload, Optional, Tuple, Callable, ValuesView
from pyspark.sql.connect.column import Column
from pyspark.sql.connect.expressions import (
@@ -28,8 +29,7 @@ from pyspark.sql.connect.expressions import (
SQLExpression,
LambdaFunction,
)
-
-from typing import Any, TYPE_CHECKING, Union, List, overload, Optional, Tuple, Callable, ValuesView
+from pyspark.sql import functions as pysparkfuncs
if TYPE_CHECKING:
from pyspark.sql.connect._typing import ColumnOrName
@@ -174,79 +174,25 @@ def lit(col: Any) -> Column:
return Column(LiteralExpression(col, dataType))
-def bitwiseNOT(col: "ColumnOrName") -> Column:
- """
- Computes bitwise not.
+lit.__doc__ = pysparkfuncs.lit.__doc__
- .. versionadded:: 3.4.0
- .. deprecated:: 3.4.0
- Use :func:`bitwise_not` instead.
- """
+def bitwiseNOT(col: "ColumnOrName") -> Column:
warnings.warn("Deprecated in 3.4, use bitwise_not instead.", FutureWarning)
return bitwise_not(col)
-def bitwise_not(col: "ColumnOrName") -> Column:
- """
- Computes bitwise not.
-
- .. versionadded:: 3.4.0
+bitwiseNOT.__doc__ = pysparkfuncs.bitwiseNOT.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(bitwise_not(lit(0))).show()
- +---+
- | ~0|
- +---+
- | -1|
- +---+
- >>> df.select(bitwise_not(lit(1))).show()
- +---+
- | ~1|
- +---+
- | -2|
- +---+
- """
+def bitwise_not(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("~", col)
-def broadcast(df: "DataFrame") -> "DataFrame":
- """
- Marks a DataFrame as small enough for use in broadcast joins.
+bitwise_not.__doc__ = pysparkfuncs.bitwise_not.__doc__
- .. versionadded:: 3.4.0
- Returns
- -------
- :class:`~pyspark.sql.DataFrame`
- DataFrame marked as ready for broadcast join.
-
- Examples
- --------
- >>> from pyspark.sql import types
- >>> df = spark.createDataFrame([1, 2, 3, 3, 4], types.IntegerType())
- >>> df_small = spark.range(3)
- >>> df_b = broadcast(df_small)
- >>> df.join(df_b, df.value == df_small.id).show()
- +-----+---+
- |value| id|
- +-----+---+
- | 1| 1|
- | 2| 2|
- +-----+---+
- """
+def broadcast(df: "DataFrame") -> "DataFrame":
from pyspark.sql.connect.dataframe import DataFrame
if not isinstance(df, DataFrame):
@@ -254,416 +200,104 @@ def broadcast(df: "DataFrame") -> "DataFrame":
return df.hint("broadcast")
-def coalesce(*cols: "ColumnOrName") -> Column:
- """Returns the first column that is not null.
-
- .. versionadded:: 3.4.0
+broadcast.__doc__ = pysparkfuncs.broadcast.__doc__
- Parameters
- ----------
- cols : :class:`~pyspark.sql.Column` or str
- list of columns to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- value of the first column that is not null.
-
- Examples
- --------
- >>> cDf = spark.createDataFrame([(None, None), (1, None), (None, 2)], ("a", "b"))
- >>> cDf.show()
- +----+----+
- | a| b|
- +----+----+
- |null|null|
- | 1|null|
- |null| 2|
- +----+----+
-
- >>> cDf.select(coalesce(cDf["a"], cDf["b"])).show()
- +--------------+
- |coalesce(a, b)|
- +--------------+
- | null|
- | 1|
- | 2|
- +--------------+
-
- >>> cDf.select('*', coalesce(cDf["a"], lit(0.0))).show()
- +----+----+----------------+
- | a| b|coalesce(a, 0.0)|
- +----+----+----------------+
- |null|null| 0.0|
- | 1|null| 1.0|
- |null| 2| 0.0|
- +----+----+----------------+
- """
+def coalesce(*cols: "ColumnOrName") -> Column:
return _invoke_function_over_columns("coalesce", *cols)
-def expr(str: str) -> Column:
- """Parses the expression string into the column that it represents
-
- .. versionadded:: 3.4.0
+coalesce.__doc__ = pysparkfuncs.coalesce.__doc__
- Parameters
- ----------
- str : str
- expression defined in string.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- column representing the expression.
-
- Examples
- --------
- >>> df = spark.createDataFrame([["Alice"], ["Bob"]], ["name"])
- >>> df.select("name", expr("length(name)")).show()
- +-----+------------+
- | name|length(name)|
- +-----+------------+
- |Alice| 5|
- | Bob| 3|
- +-----+------------+
- """
+def expr(str: str) -> Column:
return Column(SQLExpression(str))
-def greatest(*cols: "ColumnOrName") -> Column:
- """
- Returns the greatest value of the list of column names, skipping null values.
- This function takes at least 2 parameters. It will return null if all parameters are null.
-
- .. versionadded:: 3.4.0
+expr.__doc__ = pysparkfuncs.expr.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- columns to check for gratest value.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- gratest value.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(1, 4, 3)], ['a', 'b', 'c'])
- >>> df.select(greatest(df.a, df.b, df.c).alias("greatest")).collect()
- [Row(greatest=4)]
- """
+def greatest(*cols: "ColumnOrName") -> Column:
if len(cols) < 2:
raise ValueError("greatest should take at least two columns")
return _invoke_function_over_columns("greatest", *cols)
-def input_file_name() -> Column:
- """
- Creates a string column for the file name of the current Spark task.
+greatest.__doc__ = pysparkfuncs.greatest.__doc__
- .. versionadded:: 3.4.0
- Returns
- -------
- :class:`~pyspark.sql.Column`
- file names.
-
- Examples
- --------
- >>> import os
- >>> path = os.path.abspath(__file__)
- >>> df = spark.read.text(path)
- >>> df.select(input_file_name()).first()
- Row(input_file_name()='file:///...')
- """
+def input_file_name() -> Column:
return _invoke_function("input_file_name")
-def least(*cols: "ColumnOrName") -> Column:
- """
- Returns the least value of the list of column names, skipping null values.
- This function takes at least 2 parameters. It will return null if all parameters are null.
-
- .. versionadded:: 3.4.0
+input_file_name.__doc__ = pysparkfuncs.input_file_name.__doc__
- Parameters
- ----------
- cols : :class:`~pyspark.sql.Column` or str
- column names or columns to be compared
- Returns
- -------
- :class:`~pyspark.sql.Column`
- least value.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(1, 4, 3)], ['a', 'b', 'c'])
- >>> df.select(least(df.a, df.b, df.c).alias("least")).collect()
- [Row(least=1)]
- """
+def least(*cols: "ColumnOrName") -> Column:
if len(cols) < 2:
raise ValueError("least should take at least two columns")
return _invoke_function_over_columns("least", *cols)
-def isnan(col: "ColumnOrName") -> Column:
- """An expression that returns true if the column is NaN.
-
- .. versionadded:: 3.4.0
+least.__doc__ = pysparkfuncs.least.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- True if value is NaN and False otherwise.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(1.0, float('nan')), (float('nan'), 2.0)], ("a", "b"))
- >>> df.select("a", "b", isnan("a").alias("r1"), isnan(df.b).alias("r2")).show()
- +---+---+-----+-----+
- | a| b| r1| r2|
- +---+---+-----+-----+
- |1.0|NaN|false| true|
- |NaN|2.0| true|false|
- +---+---+-----+-----+
- """
+def isnan(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("isnan", col)
-def isnull(col: "ColumnOrName") -> Column:
- """An expression that returns true if the column is null.
-
- .. versionadded:: 3.4.0
+isnan.__doc__ = pysparkfuncs.isnan.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- True if value is null and False otherwise.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(1, None), (None, 2)], ("a", "b"))
- >>> df.select("a", "b", isnull("a").alias("r1"), isnull(df.b).alias("r2")).show()
- +----+----+-----+-----+
- | a| b| r1| r2|
- +----+----+-----+-----+
- | 1|null|false| true|
- |null| 2| true|false|
- +----+----+-----+-----+
- """
+def isnull(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("isnull", col)
-def monotonically_increasing_id() -> Column:
- """A column that generates monotonically increasing 64-bit integers.
-
- The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive.
- The current implementation puts the partition ID in the upper 31 bits, and the record number
- within each partition in the lower 33 bits. The assumption is that the data frame has
- less than 1 billion partitions, and each partition has less than 8 billion records.
-
- .. versionadded:: 3.4.0
-
- Notes
- -----
- The function is non-deterministic because its result depends on partition IDs.
+isnull.__doc__ = pysparkfuncs.isnull.__doc__
- As an example, consider a :class:`DataFrame` with two partitions, each with 3 records.
- This expression would return the following IDs:
- 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- last value of the group.
-
- Examples
- --------
- >>> df0 = sc.parallelize(range(2), 2).mapPartitions(lambda x: [(1,), (2,), (3,)]).toDF(['col1'])
- >>> df0.select(monotonically_increasing_id().alias('id')).collect()
- [Row(id=0), Row(id=1), Row(id=2), Row(id=8589934592), Row(id=8589934593), Row(id=8589934594)]
- """
+def monotonically_increasing_id() -> Column:
return _invoke_function("monotonically_increasing_id")
-def nanvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
- """Returns col1 if it is not NaN, or col2 if col1 is NaN.
-
- Both inputs should be floating point columns (:class:`DoubleType` or :class:`FloatType`).
-
- .. versionadded:: 3.4.0
+monotonically_increasing_id.__doc__ = pysparkfuncs.monotonically_increasing_id.__doc__
- Parameters
- ----------
- col1 : :class:`~pyspark.sql.Column` or str
- first column to check.
- col2 : :class:`~pyspark.sql.Column` or str
- second column to return if first is NaN.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- value from first column or second if first is NaN .
-
- Examples
- --------
- >>> df = spark.createDataFrame([(1.0, float('nan')), (float('nan'), 2.0)], ("a", "b"))
- >>> df.select(nanvl("a", "b").alias("r1"), nanvl(df.a, df.b).alias("r2")).collect()
- [Row(r1=1.0, r2=1.0), Row(r1=2.0, r2=2.0)]
- """
+def nanvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
return _invoke_function_over_columns("nanvl", col1, col2)
-def rand(seed: Optional[int] = None) -> Column:
- """Generates a random column with independent and identically distributed (i.i.d.) samples
- uniformly distributed in [0.0, 1.0).
-
- .. versionadded:: 3.4.0
-
- Notes
- -----
- The function is non-deterministic in general case.
+nanvl.__doc__ = pysparkfuncs.nanvl.__doc__
- Parameters
- ----------
- seed : int (default: None)
- seed value for random generator.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- random values.
-
- Examples
- --------
- >>> df = spark.range(2)
- >>> df.withColumn('rand', rand(seed=42) * 3).show() # doctest: +SKIP
- +---+------------------+
- | id| rand|
- +---+------------------+
- | 0|1.4385751892400076|
- | 1|1.7082186019706387|
- +---+------------------+
- """
+def rand(seed: Optional[int] = None) -> Column:
if seed is not None:
return _invoke_function("rand", lit(seed))
else:
return _invoke_function("rand")
-def randn(seed: Optional[int] = None) -> Column:
- """Generates a column with independent and identically distributed (i.i.d.) samples from
- the standard normal distribution.
-
- .. versionadded:: 3.4.0
+rand.__doc__ = pysparkfuncs.rand.__doc__
- Notes
- -----
- The function is non-deterministic in general case.
-
- Parameters
- ----------
- seed : int (default: None)
- seed value for random generator.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- random values.
-
- Examples
- --------
- >>> df = spark.range(2)
- >>> df.withColumn('randn', randn(seed=42)).show() # doctest: +SKIP
- +---+--------------------+
- | id| randn|
- +---+--------------------+
- | 0|-0.04167221574820542|
- | 1| 0.15241403986452778|
- +---+--------------------+
- """
+def randn(seed: Optional[int] = None) -> Column:
if seed is not None:
return _invoke_function("randn", lit(seed))
else:
return _invoke_function("randn")
-def spark_partition_id() -> Column:
- """A column for partition ID.
-
- .. versionadded:: 3.4.0
+randn.__doc__ = pysparkfuncs.randn.__doc__
- Notes
- -----
- This is non deterministic because it depends on data partitioning and task scheduling.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- partition id the record belongs to.
-
- Examples
- --------
- >>> df = spark.range(2)
- >>> df.repartition(1).select(spark_partition_id().alias("pid")).collect()
- [Row(pid=0), Row(pid=0)]
- """
+def spark_partition_id() -> Column:
return _invoke_function("spark_partition_id")
-def when(condition: Column, value: Any) -> Column:
- """Evaluates a list of conditions and returns one of multiple possible result expressions.
- If :func:`pyspark.sql.Column.otherwise` is not invoked, None is returned for unmatched
- conditions.
-
- .. versionadded:: 3.4.0
+spark_partition_id.__doc__ = pysparkfuncs.spark_partition_id.__doc__
- Parameters
- ----------
- condition : :class:`~pyspark.sql.Column`
- a boolean :class:`~pyspark.sql.Column` expression.
- value :
- a literal value, or a :class:`~pyspark.sql.Column` expression.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- column representing when expression.
-
- Examples
- --------
- >>> df = spark.range(3)
- >>> df.select(when(df['id'] == 2, 3).otherwise(4).alias("age")).show()
- +---+
- |age|
- +---+
- | 4|
- | 4|
- | 3|
- +---+
-
- >>> df.select(when(df.id == 2, df.id + 1).alias("age")).show()
- +----+
- | age|
- +----+
- |null|
- |null|
- | 3|
- +----+
- """
+def when(condition: Column, value: Any) -> Column:
# Explicitly not using ColumnOrName type here to make reading condition less opaque
if not isinstance(condition, Column):
raise TypeError("condition should be a Column")
@@ -673,2267 +307,569 @@ def when(condition: Column, value: Any) -> Column:
return Column(CaseWhen(branches=[(condition._expr, value_col._expr)], else_value=None))
-# Sort Functions
-
+when.__doc__ = pysparkfuncs.when.__doc__
-def asc(col: "ColumnOrName") -> Column:
- """
- Returns a sort expression based on the ascending order of the given column name.
- .. versionadded:: 3.4.0
+# Sort Functions
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to sort by in the ascending order.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column specifying the order.
-
- Examples
- --------
- Sort by the column 'id' in the descending order.
-
- >>> df = spark.range(5)
- >>> df = df.sort(desc("id"))
- >>> df.show()
- +---+
- | id|
- +---+
- | 4|
- | 3|
- | 2|
- | 1|
- | 0|
- +---+
-
- Sort by the column 'id' in the ascending order.
-
- >>> df.orderBy(asc("id")).show()
- +---+
- | id|
- +---+
- | 0|
- | 1|
- | 2|
- | 3|
- | 4|
- +---+
- """
+def asc(col: "ColumnOrName") -> Column:
return _to_col(col).asc()
-def asc_nulls_first(col: "ColumnOrName") -> Column:
- """
- Returns a sort expression based on the ascending order of the given
- column name, and null values return before non-null values.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to sort by in the ascending order.
+asc.__doc__ = pysparkfuncs.asc.__doc__
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column specifying the order.
-
- Examples
- --------
- >>> df1 = spark.createDataFrame([(1, "Bob"),
- ... (0, None),
- ... (2, "Alice")], ["age", "name"])
- >>> df1.sort(asc_nulls_first(df1.name)).show()
- +---+-----+
- |age| name|
- +---+-----+
- | 0| null|
- | 2|Alice|
- | 1| Bob|
- +---+-----+
- """
+def asc_nulls_first(col: "ColumnOrName") -> Column:
return _to_col(col).asc_nulls_first()
-def asc_nulls_last(col: "ColumnOrName") -> Column:
- """
- Returns a sort expression based on the ascending order of the given
- column name, and null values appear after non-null values.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to sort by in the ascending order.
+asc_nulls_first.__doc__ = pysparkfuncs.asc_nulls_first.__doc__
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column specifying the order.
-
- Examples
- --------
- >>> df1 = spark.createDataFrame([(0, None),
- ... (1, "Bob"),
- ... (2, "Alice")], ["age", "name"])
- >>> df1.sort(asc_nulls_last(df1.name)).show()
- +---+-----+
- |age| name|
- +---+-----+
- | 2|Alice|
- | 1| Bob|
- | 0| null|
- +---+-----+
- """
+def asc_nulls_last(col: "ColumnOrName") -> Column:
return _to_col(col).asc_nulls_last()
-def desc(col: "ColumnOrName") -> Column:
- """
- Returns a sort expression based on the descending order of the given column name.
-
- .. versionadded:: 3.4.0
+asc_nulls_last.__doc__ = pysparkfuncs.asc_nulls_last.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to sort by in the descending order.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column specifying the order.
-
- Examples
- --------
- Sort by the column 'id' in the descending order.
-
- >>> spark.range(5).orderBy(desc("id")).show()
- +---+
- | id|
- +---+
- | 4|
- | 3|
- | 2|
- | 1|
- | 0|
- +---+
- """
+def desc(col: "ColumnOrName") -> Column:
return _to_col(col).desc()
-def desc_nulls_first(col: "ColumnOrName") -> Column:
- """
- Returns a sort expression based on the descending order of the given
- column name, and null values appear before non-null values.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to sort by in the descending order.
+desc.__doc__ = pysparkfuncs.desc.__doc__
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column specifying the order.
-
- Examples
- --------
- >>> df1 = spark.createDataFrame([(0, None),
- ... (1, "Bob"),
- ... (2, "Alice")], ["age", "name"])
- >>> df1.sort(desc_nulls_first(df1.name)).show()
- +---+-----+
- |age| name|
- +---+-----+
- | 0| null|
- | 1| Bob|
- | 2|Alice|
- +---+-----+
- """
+def desc_nulls_first(col: "ColumnOrName") -> Column:
return _to_col(col).desc_nulls_first()
-def desc_nulls_last(col: "ColumnOrName") -> Column:
- """
- Returns a sort expression based on the descending order of the given
- column name, and null values appear after non-null values.
+desc_nulls_first.__doc__ = pysparkfuncs.desc_nulls_first.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to sort by in the descending order.
+def desc_nulls_last(col: "ColumnOrName") -> Column:
+ return _to_col(col).desc_nulls_last()
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column specifying the order.
-
- Examples
- --------
- >>> df1 = spark.createDataFrame([(0, None),
- ... (1, "Bob"),
- ... (2, "Alice")], ["age", "name"])
- >>> df1.sort(desc_nulls_last(df1.name)).show()
- +---+-----+
- |age| name|
- +---+-----+
- | 1| Bob|
- | 2|Alice|
- | 0| null|
- +---+-----+
- """
- return _to_col(col).desc_nulls_last()
+desc_nulls_last.__doc__ = pysparkfuncs.desc_nulls_last.__doc__
# Math Functions
def abs(col: "ColumnOrName") -> Column:
- """
- Computes the absolute value.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("abs", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- column for computed results.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(abs(lit(-1))).show()
- +-------+
- |abs(-1)|
- +-------+
- | 1|
- +-------+
- """
- return _invoke_function_over_columns("abs", col)
+abs.__doc__ = pysparkfuncs.abs.__doc__
def acos(col: "ColumnOrName") -> Column:
- """
- Computes inverse cosine of the input column.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("acos", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- inverse cosine of `col`, as if computed by `java.lang.Math.acos()`
-
- Examples
- --------
- >>> df = spark.range(1, 3)
- >>> df.select(acos(df.id)).show()
- +--------+
- |ACOS(id)|
- +--------+
- | 0.0|
- | NaN|
- +--------+
- """
- return _invoke_function_over_columns("acos", col)
+acos.__doc__ = pysparkfuncs.acos.__doc__
def acosh(col: "ColumnOrName") -> Column:
- """
- Computes inverse hyperbolic cosine of the input column.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("acosh", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.range(2)
- >>> df.select(acosh(col("id"))).show()
- +---------+
- |ACOSH(id)|
- +---------+
- | NaN|
- | 0.0|
- +---------+
- """
- return _invoke_function_over_columns("acosh", col)
+acosh.__doc__ = pysparkfuncs.acosh.__doc__
def asin(col: "ColumnOrName") -> Column:
- """
- Computes inverse sine of the input column.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("asin", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- inverse sine of `col`, as if computed by `java.lang.Math.asin()`
-
- Examples
- --------
- >>> df = spark.createDataFrame([(0,), (2,)])
- >>> df.select(asin(df.schema.fieldNames()[0])).show()
- +--------+
- |ASIN(_1)|
- +--------+
- | 0.0|
- | NaN|
- +--------+
- """
- return _invoke_function_over_columns("asin", col)
+asin.__doc__ = pysparkfuncs.asin.__doc__
def asinh(col: "ColumnOrName") -> Column:
- """
- Computes inverse hyperbolic sine of the input column.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("asinh", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(asinh(col("id"))).show()
- +---------+
- |ASINH(id)|
- +---------+
- | 0.0|
- +---------+
- """
- return _invoke_function_over_columns("asinh", col)
+asinh.__doc__ = pysparkfuncs.asinh.__doc__
def atan(col: "ColumnOrName") -> Column:
- """
- Compute inverse tangent of the input column.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("atan", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- inverse tangent of `col`, as if computed by `java.lang.Math.atan()`
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(atan(df.id)).show()
- +--------+
- |ATAN(id)|
- +--------+
- | 0.0|
- +--------+
- """
- return _invoke_function_over_columns("atan", col)
+atan.__doc__ = pysparkfuncs.atan.__doc__
def atan2(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column:
- """
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col1 : str, :class:`~pyspark.sql.Column` or float
- coordinate on y-axis
- col2 : str, :class:`~pyspark.sql.Column` or float
- coordinate on x-axis
+ return _invoke_binary_math_function("atan2", col1, col2)
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the `theta` component of the point
- (`r`, `theta`)
- in polar coordinates that corresponds to the point
- (`x`, `y`) in Cartesian coordinates,
- as if computed by `java.lang.Math.atan2()`
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(atan2(lit(1), lit(2))).first()
- Row(ATAN2(1, 2)=0.46364...)
- """
- return _invoke_binary_math_function("atan2", col1, col2)
+atan2.__doc__ = pysparkfuncs.atan2.__doc__
def atanh(col: "ColumnOrName") -> Column:
- """
- Computes inverse hyperbolic tangent of the input column.
+ return _invoke_function_over_columns("atanh", col)
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(0,), (2,)], schema=["numbers"])
- >>> df.select(atanh(df["numbers"])).show()
- +--------------+
- |ATANH(numbers)|
- +--------------+
- | 0.0|
- | NaN|
- +--------------+
- """
- return _invoke_function_over_columns("atanh", col)
+atanh.__doc__ = pysparkfuncs.atanh.__doc__
def bin(col: "ColumnOrName") -> Column:
- """Returns the string representation of the binary value of the given column.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- binary representation of given value as string.
-
- Examples
- --------
- >>> df = spark.createDataFrame([2,5], "INT")
- >>> df.select(bin(df.value).alias('c')).collect()
- [Row(c='10'), Row(c='101')]
- """
return _invoke_function_over_columns("bin", col)
-def bround(col: "ColumnOrName", scale: int = 0) -> Column:
- """
- Round the given value to `scale` decimal places using HALF_EVEN rounding mode if `scale` >= 0
- or at integral part when `scale` < 0.
+bin.__doc__ = pysparkfuncs.bin.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- input column to round.
- scale : int optional default 0
- scale value.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- rounded values.
-
- Examples
- --------
- >>> spark.createDataFrame([(2.5,)], ['a']).select(bround('a', 0).alias('r')).collect()
- [Row(r=2.0)]
- """
+def bround(col: "ColumnOrName", scale: int = 0) -> Column:
return _invoke_function("bround", _to_col(col), lit(scale))
-def cbrt(col: "ColumnOrName") -> Column:
- """
- Computes the cube-root of the given value.
-
- .. versionadded:: 3.4.0
+bround.__doc__ = pysparkfuncs.bround.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(cbrt(lit(27))).show()
- +--------+
- |CBRT(27)|
- +--------+
- | 3.0|
- +--------+
- """
+def cbrt(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("cbrt", col)
-def ceil(col: "ColumnOrName") -> Column:
- """
- Computes the ceiling of the given value.
+cbrt.__doc__ = pysparkfuncs.cbrt.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(ceil(lit(-0.1))).show()
- +----------+
- |CEIL(-0.1)|
- +----------+
- | 0|
- +----------+
- """
+def ceil(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("ceil", col)
-def conv(col: "ColumnOrName", fromBase: int, toBase: int) -> Column:
- """
- Convert a number in a string column from one base to another.
-
- .. versionadded:: 3.4.0
+ceil.__doc__ = pysparkfuncs.ceil.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- a column to convert base for.
- fromBase: int
- from base number.
- toBase: int
- to base number.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- logariphm of given value.
-
- Examples
- --------
- >>> df = spark.createDataFrame([("010101",)], ['n'])
- >>> df.select(conv(df.n, 2, 16).alias('hex')).collect()
- [Row(hex='15')]
- """
+def conv(col: "ColumnOrName", fromBase: int, toBase: int) -> Column:
return _invoke_function("conv", _to_col(col), lit(fromBase), lit(toBase))
-def cos(col: "ColumnOrName") -> Column:
- """
- Computes cosine of the input column.
-
- .. versionadded:: 3.4.0
+conv.__doc__ = pysparkfuncs.conv.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- angle in radians
- Returns
- -------
- :class:`~pyspark.sql.Column`
- cosine of the angle, as if computed by `java.lang.Math.cos()`.
-
- Examples
- --------
- >>> import math
- >>> df = spark.range(1)
- >>> df.select(cos(lit(math.pi))).first()
- Row(COS(3.14159...)=-1.0)
- """
+def cos(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("cos", col)
-def cosh(col: "ColumnOrName") -> Column:
- """
- Computes hyperbolic cosine of the input column.
-
- .. versionadded:: 3.4.0
+cos.__doc__ = pysparkfuncs.cos.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- hyperbolic angle
- Returns
- -------
- :class:`~pyspark.sql.Column`
- hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh()`
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(cosh(lit(1))).first()
- Row(COSH(1)=1.54308...)
- """
+def cosh(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("cosh", col)
-def cot(col: "ColumnOrName") -> Column:
- """
- Computes cotangent of the input column.
-
- .. versionadded:: 3.4.0
+cosh.__doc__ = pysparkfuncs.cosh.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- angle in radians.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- cotangent of the angle.
-
- Examples
- --------
- >>> import math
- >>> df = spark.range(1)
- >>> df.select(cot(lit(math.radians(45)))).first()
- Row(COT(0.78539...)=1.00000...)
- """
+def cot(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("cot", col)
-def csc(col: "ColumnOrName") -> Column:
- """
- Computes cosecant of the input column.
+cot.__doc__ = pysparkfuncs.cot.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- angle in radians.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- cosecant of the angle.
-
- Examples
- --------
- >>> import math
- >>> df = spark.range(1)
- >>> df.select(csc(lit(math.radians(90)))).first()
- Row(CSC(1.57079...)=1.0)
- """
+def csc(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("csc", col)
-def degrees(col: "ColumnOrName") -> Column:
- """
- Converts an angle measured in radians to an approximately equivalent angle
- measured in degrees.
+csc.__doc__ = pysparkfuncs.csc.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- angle in radians
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- angle in degrees, as if computed by `java.lang.Math.toDegrees()`
-
- Examples
- --------
- >>> import math
- >>> df = spark.range(1)
- >>> df.select(degrees(lit(math.pi))).first()
- Row(DEGREES(3.14159...)=180.0)
- """
+def degrees(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("degrees", col)
-def exp(col: "ColumnOrName") -> Column:
- """
- Computes the exponential of the given value.
-
- .. versionadded:: 3.4.0
+degrees.__doc__ = pysparkfuncs.degrees.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- column to calculate exponential for.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- exponential of the given value.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(exp(lit(0))).show()
- +------+
- |EXP(0)|
- +------+
- | 1.0|
- +------+
- """
+def exp(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("exp", col)
-def expm1(col: "ColumnOrName") -> Column:
- """
- Computes the exponential of the given value minus one.
+exp.__doc__ = pysparkfuncs.exp.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- column to calculate exponential for.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- exponential less one.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(expm1(lit(1))).first()
- Row(EXPM1(1)=1.71828...)
- """
+def expm1(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("expm1", col)
-def factorial(col: "ColumnOrName") -> Column:
- """
- Computes the factorial of the given value.
+expm1.__doc__ = pysparkfuncs.expm1.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- a column to calculate factorial for.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- factorial of given value.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(5,)], ['n'])
- >>> df.select(factorial(df.n).alias('f')).collect()
- [Row(f=120)]
- """
+def factorial(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("factorial", col)
-def floor(col: "ColumnOrName") -> Column:
- """
- Computes the floor of the given value.
-
- .. versionadded:: 3.4.0
+factorial.__doc__ = pysparkfuncs.factorial.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- column to find floor for.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- neares integer that is less than or equal to given value.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(floor(lit(2.5))).show()
- +----------+
- |FLOOR(2.5)|
- +----------+
- | 2|
- +----------+
- """
+def floor(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("floor", col)
-def hex(col: "ColumnOrName") -> Column:
- """Computes hex value of the given column, which could be :class:`pyspark.sql.types.StringType`,
- :class:`pyspark.sql.types.BinaryType`, :class:`pyspark.sql.types.IntegerType` or
- :class:`pyspark.sql.types.LongType`.
+floor.__doc__ = pysparkfuncs.floor.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- hexadecimal representation of given value as string.
-
- Examples
- --------
- >>> spark.createDataFrame([('ABC', 3)], ['a', 'b']).select(hex('a'), hex('b')).collect()
- [Row(hex(a)='414243', hex(b)='3')]
- """
+def hex(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("hex", col)
-def hypot(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column:
- """
- Computes ``sqrt(a^2 + b^2)`` without intermediate overflow or underflow.
-
- .. versionadded:: 3.4.0
+hex.__doc__ = pysparkfuncs.hex.__doc__
- Parameters
- ----------
- col1 : str, :class:`~pyspark.sql.Column` or float
- a leg.
- col2 : str, :class:`~pyspark.sql.Column` or float
- b leg.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- length of the hypotenuse.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(hypot(lit(1), lit(2))).first()
- Row(HYPOT(1, 2)=2.23606...)
- """
+def hypot(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column:
return _invoke_binary_math_function("hypot", col1, col2)
-def log(col: "ColumnOrName") -> Column:
- """
- Computes the natural logarithm of the given value.
+hypot.__doc__ = pysparkfuncs.hypot.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- column to calculate natural logarithm for.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- natural logarithm of the given value.
-
- Examples
- --------
- >>> import math
- >>> df = spark.range(1)
- >>> df.select(log(lit(math.e))).first()
- Row(ln(2.71828...)=1.0)
- """
+def log(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("ln", col)
-def log10(col: "ColumnOrName") -> Column:
- """
- Computes the logarithm of the given value in Base 10.
-
- .. versionadded:: 3.4.0
+log.__doc__ = pysparkfuncs.log.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- column to calculate logarithm for.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- logarithm of the given value in Base 10.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(log10(lit(100))).show()
- +----------+
- |LOG10(100)|
- +----------+
- | 2.0|
- +----------+
- """
+def log10(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("log10", col)
-def log1p(col: "ColumnOrName") -> Column:
- """
- Computes the natural logarithm of the "given value plus one".
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- column to calculate natural logarithm for.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- natural logarithm of the "given value plus one".
-
- Examples
- --------
- >>> import math
- >>> df = spark.range(1)
- >>> df.select(log1p(lit(math.e))).first()
- Row(LOG1P(2.71828...)=1.31326...)
+log10.__doc__ = pysparkfuncs.log10.__doc__
- Same as:
- >>> df.select(log(lit(math.e+1))).first()
- Row(ln(3.71828...)=1.31326...)
- """
+def log1p(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("log1p", col)
-def log2(col: "ColumnOrName") -> Column:
- """Returns the base-2 logarithm of the argument.
-
- .. versionadded:: 3.4.0
+log1p.__doc__ = pysparkfuncs.log1p.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- a column to calculate logariphm for.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- logariphm of given value.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(4,)], ['a'])
- >>> df.select(log2('a').alias('log2')).show()
- +----+
- |log2|
- +----+
- | 2.0|
- +----+
- """
+def log2(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("log2", col)
-def pmod(dividend: Union["ColumnOrName", float], divisor: Union["ColumnOrName", float]) -> Column:
- """
- Returns the positive value of dividend mod divisor.
-
- .. versionadded:: 3.4.0
+log2.__doc__ = pysparkfuncs.log2.__doc__
- Parameters
- ----------
- dividend : str, :class:`~pyspark.sql.Column` or float
- the column that contains dividend, or the specified dividend value
- divisor : str, :class:`~pyspark.sql.Column` or float
- the column that contains divisor, or the specified divisor value
- Returns
- -------
- :class:`~pyspark.sql.Column`
- positive value of dividend mod divisor.
-
- Examples
- --------
- >>> from pyspark.sql.functions import pmod
- >>> df = spark.createDataFrame([
- ... (1.0, float('nan')), (float('nan'), 2.0), (10.0, 3.0),
- ... (float('nan'), float('nan')), (-3.0, 4.0), (-10.0, 3.0),
- ... (-5.0, -6.0), (7.0, -8.0), (1.0, 2.0)],
- ... ("a", "b"))
- >>> df.select(pmod("a", "b")).show()
- +----------+
- |pmod(a, b)|
- +----------+
- | NaN|
- | NaN|
- | 1.0|
- | NaN|
- | 1.0|
- | 2.0|
- | -5.0|
- | 7.0|
- | 1.0|
- +----------+
- """
+def pmod(dividend: Union["ColumnOrName", float], divisor: Union["ColumnOrName", float]) -> Column:
return _invoke_binary_math_function("pmod", dividend, divisor)
-def pow(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column:
- """
- Returns the value of the first argument raised to the power of the second argument.
-
- .. versionadded:: 3.4.0
+pmod.__doc__ = pysparkfuncs.pmod.__doc__
- Parameters
- ----------
- col1 : str, :class:`~pyspark.sql.Column` or float
- the base number.
- col2 : str, :class:`~pyspark.sql.Column` or float
- the exponent number.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the base rased to the power the argument.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(pow(lit(3), lit(2))).first()
- Row(POWER(3, 2)=9.0)
- """
+def pow(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column:
return _invoke_binary_math_function("power", col1, col2)
-def radians(col: "ColumnOrName") -> Column:
- """
- Converts an angle measured in degrees to an approximately equivalent angle
- measured in radians.
+pow.__doc__ = pysparkfuncs.pow.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- angle in degrees
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- angle in radians, as if computed by `java.lang.Math.toRadians()`
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(radians(lit(180))).first()
- Row(RADIANS(180)=3.14159...)
- """
+def radians(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("radians", col)
-def rint(col: "ColumnOrName") -> Column:
- """
- Returns the double value that is closest in value to the argument and
- is equal to a mathematical integer.
-
- .. versionadded:: 3.4.0
+radians.__doc__ = pysparkfuncs.radians.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(rint(lit(10.6))).show()
- +----------+
- |rint(10.6)|
- +----------+
- | 11.0|
- +----------+
-
- >>> df.select(rint(lit(10.3))).show()
- +----------+
- |rint(10.3)|
- +----------+
- | 10.0|
- +----------+
- """
+def rint(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("rint", col)
-def round(col: "ColumnOrName", scale: int = 0) -> Column:
- """
- Round the given value to `scale` decimal places using HALF_UP rounding mode if `scale` >= 0
- or at integral part when `scale` < 0.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- input column to round.
- scale : int optional default 0
- scale value.
+rint.__doc__ = pysparkfuncs.rint.__doc__
- Returns
- -------
- :class:`~pyspark.sql.Column`
- rounded values.
- Examples
- --------
- >>> spark.createDataFrame([(2.5,)], ['a']).select(round('a', 0).alias('r')).collect()
- [Row(r=3.0)]
- """
+def round(col: "ColumnOrName", scale: int = 0) -> Column:
return _invoke_function("round", _to_col(col), lit(scale))
-def sec(col: "ColumnOrName") -> Column:
- """
- Computes secant of the input column.
-
- .. versionadded:: 3.4.0
+round.__doc__ = pysparkfuncs.round.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- Angle in radians
- Returns
- -------
- :class:`~pyspark.sql.Column`
- Secant of the angle.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(sec(lit(1.5))).first()
- Row(SEC(1.5)=14.13683...)
- """
+def sec(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("sec", col)
-def shiftLeft(col: "ColumnOrName", numBits: int) -> Column:
- """Shift the given value numBits left.
+sec.__doc__ = pysparkfuncs.sec.__doc__
- .. versionadded:: 3.4.0
- .. deprecated:: 3.4.0
- Use :func:`shiftleft` instead.
- """
+def shiftLeft(col: "ColumnOrName", numBits: int) -> Column:
warnings.warn("Deprecated in 3.4, use shiftleft instead.", FutureWarning)
return shiftleft(col, numBits)
-def shiftleft(col: "ColumnOrName", numBits: int) -> Column:
- """Shift the given value numBits left.
-
- .. versionadded:: 3.4.0
+shiftLeft.__doc__ = pysparkfuncs.shiftLeft.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- input column of values to shift.
- numBits : int
- number of bits to shift.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- shifted value.
-
- Examples
- --------
- >>> spark.createDataFrame([(21,)], ['a']).select(shiftleft('a', 1).alias('r')).collect()
- [Row(r=42)]
- """
+def shiftleft(col: "ColumnOrName", numBits: int) -> Column:
return _invoke_function("shiftleft", _to_col(col), lit(numBits))
-def shiftRight(col: "ColumnOrName", numBits: int) -> Column:
- """(Signed) shift the given value numBits right.
+shiftleft.__doc__ = pysparkfuncs.shiftleft.__doc__
- .. versionadded:: 3.4.0
- .. deprecated:: 3.4.0
- Use :func:`shiftright` instead.
- """
+def shiftRight(col: "ColumnOrName", numBits: int) -> Column:
warnings.warn("Deprecated in 3.4, use shiftright instead.", FutureWarning)
return shiftright(col, numBits)
-def shiftright(col: "ColumnOrName", numBits: int) -> Column:
- """(Signed) shift the given value numBits right.
+shiftRight.__doc__ = pysparkfuncs.shiftRight.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- input column of values to shift.
- numBits : int
- number of bits to shift.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- shifted values.
- Examples
- --------
- >>> spark.createDataFrame([(42,)], ['a']).select(shiftright('a', 1).alias('r')).collect()
- [Row(r=21)]
- """
+def shiftright(col: "ColumnOrName", numBits: int) -> Column:
return _invoke_function("shiftright", _to_col(col), lit(numBits))
-def shiftRightUnsigned(col: "ColumnOrName", numBits: int) -> Column:
- """Unsigned shift the given value numBits right.
+shiftright.__doc__ = pysparkfuncs.shiftright.__doc__
- .. versionadded:: 3.4.0
- .. deprecated:: 3.4.0
- Use :func:`shiftrightunsigned` instead.
- """
+def shiftRightUnsigned(col: "ColumnOrName", numBits: int) -> Column:
warnings.warn("Deprecated in 3.4, use shiftrightunsigned instead.", FutureWarning)
return shiftrightunsigned(col, numBits)
-def shiftrightunsigned(col: "ColumnOrName", numBits: int) -> Column:
- """Unsigned shift the given value numBits right.
-
- .. versionadded:: 3.4.0
+shiftRightUnsigned.__doc__ = pysparkfuncs.shiftRightUnsigned.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- input column of values to shift.
- numBits : int
- number of bits to shift.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- shifted value.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(-42,)], ['a'])
- >>> df.select(shiftrightunsigned('a', 1).alias('r')).collect()
- [Row(r=9223372036854775787)]
- """
+def shiftrightunsigned(col: "ColumnOrName", numBits: int) -> Column:
return _invoke_function("shiftrightunsigned", _to_col(col), lit(numBits))
-def signum(col: "ColumnOrName") -> Column:
- """
- Computes the signum of the given value.
-
- .. versionadded:: 3.4.0
+shiftrightunsigned.__doc__ = pysparkfuncs.shiftrightunsigned.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(signum(lit(-5))).show()
- +----------+
- |SIGNUM(-5)|
- +----------+
- | -1.0|
- +----------+
-
- >>> df.select(signum(lit(6))).show()
- +---------+
- |SIGNUM(6)|
- +---------+
- | 1.0|
- +---------+
- """
+def signum(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("signum", col)
-def sin(col: "ColumnOrName") -> Column:
- """
- Computes sine of the input column.
-
- .. versionadded:: 3.4.0
+signum.__doc__ = pysparkfuncs.signum.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- sine of the angle, as if computed by `java.lang.Math.sin()`
-
- Examples
- --------
- >>> import math
- >>> df = spark.range(1)
- >>> df.select(sin(lit(math.radians(90)))).first()
- Row(SIN(1.57079...)=1.0)
- """
+def sin(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("sin", col)
-def sinh(col: "ColumnOrName") -> Column:
- """
- Computes hyperbolic sine of the input column.
-
- .. versionadded:: 3.4.0
+sin.__doc__ = pysparkfuncs.sin.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- hyperbolic angle.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- hyperbolic sine of the given value,
- as if computed by `java.lang.Math.sinh()`
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(sinh(lit(1.1))).first()
- Row(SINH(1.1)=1.33564...)
- """
+def sinh(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("sinh", col)
-def sqrt(col: "ColumnOrName") -> Column:
- """
- Computes the square root of the specified float value.
+sinh.__doc__ = pysparkfuncs.sinh.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- column for computed results.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(sqrt(lit(4))).show()
- +-------+
- |SQRT(4)|
- +-------+
- | 2.0|
- +-------+
- """
+def sqrt(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("sqrt", col)
-def tan(col: "ColumnOrName") -> Column:
- """
- Computes tangent of the input column.
-
- .. versionadded:: 3.4.0
+sqrt.__doc__ = pysparkfuncs.sqrt.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- angle in radians
- Returns
- -------
- :class:`~pyspark.sql.Column`
- tangent of the given value, as if computed by `java.lang.Math.tan()`
-
- Examples
- --------
- >>> import math
- >>> df = spark.range(1)
- >>> df.select(tan(lit(math.radians(45)))).first()
- Row(TAN(0.78539...)=0.99999...)
- """
+def tan(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("tan", col)
-def tanh(col: "ColumnOrName") -> Column:
- """
- Computes hyperbolic tangent of the input column.
-
- .. versionadded:: 3.4.0
+tan.__doc__ = pysparkfuncs.tan.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- hyperbolic angle
- Returns
- -------
- :class:`~pyspark.sql.Column`
- hyperbolic tangent of the given value
- as if computed by `java.lang.Math.tanh()`
-
- Examples
- --------
- >>> import math
- >>> df = spark.range(1)
- >>> df.select(tanh(lit(math.radians(90)))).first()
- Row(TANH(1.57079...)=0.91715...)
- """
+def tanh(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("tanh", col)
-def toDegrees(col: "ColumnOrName") -> Column:
- """
- .. versionadded:: 3.4.0
+tanh.__doc__ = pysparkfuncs.tanh.__doc__
- .. deprecated:: 3.4.0
- Use :func:`degrees` instead.
- """
+
+def toDegrees(col: "ColumnOrName") -> Column:
warnings.warn("Deprecated in 3.4, use degrees instead.", FutureWarning)
return degrees(col)
-def toRadians(col: "ColumnOrName") -> Column:
- """
- .. versionadded:: 3.4.0
+toDegrees.__doc__ = pysparkfuncs.toDegrees.__doc__
- .. deprecated:: 3.4.0
- Use :func:`radians` instead.
- """
+
+def toRadians(col: "ColumnOrName") -> Column:
warnings.warn("Deprecated in 3.4, use radians instead.", FutureWarning)
return radians(col)
-def unhex(col: "ColumnOrName") -> Column:
- """Inverse of hex. Interprets each pair of characters as a hexadecimal number
- and converts to the byte representation of number.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
+toRadians.__doc__ = pysparkfuncs.toRadians.__doc__
- Returns
- -------
- :class:`~pyspark.sql.Column`
- string representation of given hexadecimal value.
- Examples
- --------
- >>> spark.createDataFrame([('414243',)], ['a']).select(unhex('a')).collect()
- [Row(unhex(a)=bytearray(b'ABC'))]
- """
+def unhex(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("unhex", col)
-# Aggregate Functions
+unhex.__doc__ = pysparkfuncs.unhex.__doc__
def approxCountDistinct(col: "ColumnOrName", rsd: Optional[float] = None) -> Column:
- """
- .. versionadded:: 3.4.0
-
- .. deprecated:: 3.4.0
- Use :func:`approx_count_distinct` instead.
- """
warnings.warn("Deprecated in 3.4, use approx_count_distinct instead.", FutureWarning)
return approx_count_distinct(col, rsd)
-def approx_count_distinct(col: "ColumnOrName", rsd: Optional[float] = None) -> Column:
- """Aggregate function: returns a new :class:`~pyspark.sql.Column` for approximate distinct count
- of column `col`.
+approxCountDistinct.__doc__ = pysparkfuncs.approxCountDistinct.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- rsd : float, optional
- maximum relative standard deviation allowed (default = 0.05).
- For rsd < 0.01, it is more efficient to use :func:`count_distinct`
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column of computed results.
-
- Examples
- --------
- >>> df = spark.createDataFrame([1,2,2,3], "INT")
- >>> df.agg(approx_count_distinct("value").alias('distinct_values')).show()
- +---------------+
- |distinct_values|
- +---------------+
- | 3|
- +---------------+
- """
+def approx_count_distinct(col: "ColumnOrName", rsd: Optional[float] = None) -> Column:
if rsd is None:
return _invoke_function("approx_count_distinct", _to_col(col))
else:
return _invoke_function("approx_count_distinct", _to_col(col), lit(rsd))
-def avg(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns the average of the values in a group.
+approx_count_distinct.__doc__ = pysparkfuncs.approx_count_distinct.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.range(10)
- >>> df.select(avg(col("id"))).show()
- +-------+
- |avg(id)|
- +-------+
- | 4.5|
- +-------+
- """
+def avg(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("avg", col)
-def collect_list(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns a list of objects with duplicates.
-
- .. versionadded:: 3.4.0
+avg.__doc__ = pysparkfuncs.avg.__doc__
- Notes
- -----
- The function is non-deterministic because the order of collected results depends
- on the order of the rows which may be non-deterministic after a shuffle.
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- list of objects with duplicates.
-
- Examples
- --------
- >>> df2 = spark.createDataFrame([(2,), (5,), (5,)], ('age',))
- >>> df2.agg(collect_list('age')).collect()
- [Row(collect_list(age)=[2, 5, 5])]
- """
+def collect_list(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("collect_list", col)
-def collect_set(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns a set of objects with duplicate elements eliminated.
-
- .. versionadded:: 3.4.0
-
- Notes
- -----
- The function is non-deterministic because the order of collected results depends
- on the order of the rows which may be non-deterministic after a shuffle.
+collect_list.__doc__ = pysparkfuncs.collect_list.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- list of objects with no duplicates.
-
- Examples
- --------
- >>> df2 = spark.createDataFrame([(2,), (5,), (5,)], ('age',))
- >>> df2.agg(array_sort(collect_set('age')).alias('c')).collect()
- [Row(c=[2, 5])]
- """
+def collect_set(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("collect_set", col)
-def corr(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
- """Returns a new :class:`~pyspark.sql.Column` for the Pearson Correlation Coefficient for
- ``col1`` and ``col2``.
-
- .. versionadded:: 3.4.0
+collect_set.__doc__ = pysparkfuncs.collect_set.__doc__
- Parameters
- ----------
- col1 : :class:`~pyspark.sql.Column` or str
- first column to calculate correlation.
- col1 : :class:`~pyspark.sql.Column` or str
- second column to calculate correlation.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- Pearson Correlation Coefficient of these two column values.
-
- Examples
- --------
- >>> a = range(20)
- >>> b = [2 * x for x in range(20)]
- >>> df = spark.createDataFrame(zip(a, b), ["a", "b"])
- >>> df.agg(corr("a", "b").alias('c')).collect()
- [Row(c=1.0)]
- """
+def corr(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
return _invoke_function_over_columns("corr", col1, col2)
-def count(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns the number of items in a group.
-
- .. versionadded:: 3.4.0
+corr.__doc__ = pysparkfuncs.corr.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- column for computed results.
-
- Examples
- --------
- Count by all columns (start), and by a column that does not count ``None``.
-
- >>> df = spark.createDataFrame([(None,), ("a",), ("b",), ("c",)], schema=["alphabets"])
- >>> df.select(count(expr("*")), count(df.alphabets)).show()
- +--------+----------------+
- |count(1)|count(alphabets)|
- +--------+----------------+
- | 4| 3|
- +--------+----------------+
- """
+def count(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("count", col)
-def countDistinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column:
- """Returns a new :class:`~pyspark.sql.Column` for distinct count of ``col`` or ``cols``.
+count.__doc__ = pysparkfuncs.count.__doc__
- An alias of :func:`count_distinct`, and it is encouraged to use :func:`count_distinct`
- directly.
- .. versionadded:: 3.4.0
- """
+def countDistinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column:
return count_distinct(col, *cols)
-def count_distinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column:
- """Returns a new :class:`Column` for distinct count of ``col`` or ``cols``.
-
- .. versionadded:: 3.4.0
+countDistinct.__doc__ = pysparkfuncs.countDistinct.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- first column to compute on.
- cols : :class:`~pyspark.sql.Column` or str
- other columns to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- distinct values of these two column values.
-
- Examples
- --------
- >>> from pyspark.sql import types
- >>> df1 = spark.createDataFrame([1, 1, 3], types.IntegerType())
- >>> df2 = spark.createDataFrame([1, 2], types.IntegerType())
- >>> df1.join(df2).show()
- +-----+-----+
- |value|value|
- +-----+-----+
- | 1| 1|
- | 1| 2|
- | 1| 1|
- | 1| 2|
- | 3| 1|
- | 3| 2|
- +-----+-----+
- >>> df1.join(df2).select(count_distinct(df1.value, df2.value)).show()
- +----------------------------+
- |count(DISTINCT value, value)|
- +----------------------------+
- | 4|
- +----------------------------+
- """
+def count_distinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column:
_exprs = [_to_col(c)._expr for c in [col] + list(cols)]
return Column(UnresolvedFunction("count", _exprs, is_distinct=True))
-def covar_pop(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
- """Returns a new :class:`~pyspark.sql.Column` for the population covariance of ``col1`` and
- ``col2``.
-
- .. versionadded:: 3.4.0
+count_distinct.__doc__ = pysparkfuncs.count_distinct.__doc__
- Parameters
- ----------
- col1 : :class:`~pyspark.sql.Column` or str
- first column to calculate covariance.
- col1 : :class:`~pyspark.sql.Column` or str
- second column to calculate covariance.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- covariance of these two column values.
-
- Examples
- --------
- >>> a = [1] * 10
- >>> b = [1] * 10
- >>> df = spark.createDataFrame(zip(a, b), ["a", "b"])
- >>> df.agg(covar_pop("a", "b").alias('c')).collect()
- [Row(c=0.0)]
- """
+def covar_pop(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
return _invoke_function_over_columns("covar_pop", col1, col2)
-def covar_samp(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
- """Returns a new :class:`~pyspark.sql.Column` for the sample covariance of ``col1`` and
- ``col2``.
-
- .. versionadded:: 3.4.0
+covar_pop.__doc__ = pysparkfuncs.covar_pop.__doc__
- Parameters
- ----------
- col1 : :class:`~pyspark.sql.Column` or str
- first column to calculate covariance.
- col1 : :class:`~pyspark.sql.Column` or str
- second column to calculate covariance.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- sample covariance of these two column values.
-
- Examples
- --------
- >>> a = [1] * 10
- >>> b = [1] * 10
- >>> df = spark.createDataFrame(zip(a, b), ["a", "b"])
- >>> df.agg(covar_samp("a", "b").alias('c')).collect()
- [Row(c=0.0)]
- """
+def covar_samp(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
return _invoke_function_over_columns("covar_samp", col1, col2)
-def first(col: "ColumnOrName", ignorenulls: bool = False) -> Column:
- """Aggregate function: returns the first value in a group.
-
- The function by default returns the first values it sees. It will return the first non-null
- value it sees when ignoreNulls is set to true. If all values are null, then null is returned.
-
- .. versionadded:: 3.4.0
+covar_samp.__doc__ = pysparkfuncs.covar_samp.__doc__
- Notes
- -----
- The function is non-deterministic because its results depends on the order of the
- rows which may be non-deterministic after a shuffle.
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- column to fetch first value for.
- ignorenulls : :class:`~pyspark.sql.Column` or str
- if first value is null then look for first non-null value.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- first value of the group.
-
- Examples
- --------
- >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5), ("Alice", None)], ("name", "age"))
- >>> df = df.orderBy(df.age)
- >>> df.groupby("name").agg(first("age")).orderBy("name").show()
- +-----+----------+
- | name|first(age)|
- +-----+----------+
- |Alice| null|
- | Bob| 5|
- +-----+----------+
-
- Now, to ignore any nulls we needs to set ``ignorenulls`` to `True`
-
- >>> df.groupby("name").agg(first("age", ignorenulls=True)).orderBy("name").show()
- +-----+----------+
- | name|first(age)|
- +-----+----------+
- |Alice| 2|
- | Bob| 5|
- +-----+----------+
- """
+def first(col: "ColumnOrName", ignorenulls: bool = False) -> Column:
return _invoke_function("first", _to_col(col), lit(ignorenulls))
-def grouping(col: "ColumnOrName") -> Column:
- """
- Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated
- or not, returns 1 for aggregated or 0 for not aggregated in the result set.
-
- .. versionadded:: 3.4.0
+first.__doc__ = pysparkfuncs.first.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- column to check if it's aggregated.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- returns 1 for aggregated or 0 for not aggregated in the result set.
-
- Examples
- --------
- >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5)], ("name", "age"))
- >>> df.cube("name").agg(grouping("name"), sum("age")).orderBy("name").show()
- +-----+--------------+--------+
- | name|grouping(name)|sum(age)|
- +-----+--------------+--------+
- | null| 1| 7|
- |Alice| 0| 2|
- | Bob| 0| 5|
- +-----+--------------+--------+
- """
+def grouping(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("grouping", col)
-def grouping_id(*cols: "ColumnOrName") -> Column:
- """
- Aggregate function: returns the level of grouping, equals to
-
- (grouping(c1) << (n-1)) + (grouping(c2) << (n-2)) + ... + grouping(cn)
-
- .. versionadded:: 3.4.0
+grouping.__doc__ = pysparkfuncs.grouping.__doc__
- Notes
- -----
- The list of columns should match with grouping columns exactly, or empty (means all
- the grouping columns).
-
- Parameters
- ----------
- cols : :class:`~pyspark.sql.Column` or str
- columns to check for.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- returns level of the grouping it relates to.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(1, "a", "a"),
- ... (3, "a", "a"),
- ... (4, "b", "c")], ["c1", "c2", "c3"])
- >>> df.cube("c2", "c3").agg(grouping_id(), sum("c1")).orderBy("c2", "c3").show()
- +----+----+-------------+-------+
- | c2| c3|grouping_id()|sum(c1)|
- +----+----+-------------+-------+
- |null|null| 3| 8|
- |null| a| 2| 4|
- |null| c| 2| 4|
- | a|null| 1| 4|
- | a| a| 0| 4|
- | b|null| 1| 4|
- | b| c| 0| 4|
- +----+----+-------------+-------+
- """
+def grouping_id(*cols: "ColumnOrName") -> Column:
return _invoke_function_over_columns("grouping_id", *cols)
-def kurtosis(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns the kurtosis of the values in a group.
-
- .. versionadded:: 3.4.0
+grouping_id.__doc__ = pysparkfuncs.grouping_id.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- kurtosis of given column.
-
- Examples
- --------
- >>> df = spark.createDataFrame([[1],[1],[2]], ["c"])
- >>> df.select(kurtosis(df.c)).show()
- +-----------+
- |kurtosis(c)|
- +-----------+
- | -1.5|
- +-----------+
- """
+def kurtosis(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("kurtosis", col)
-def last(col: "ColumnOrName", ignorenulls: bool = False) -> Column:
- """Aggregate function: returns the last value in a group.
-
- The function by default returns the last values it sees. It will return the last non-null
- value it sees when ignoreNulls is set to true. If all values are null, then null is returned.
+kurtosis.__doc__ = pysparkfuncs.kurtosis.__doc__
- .. versionadded:: 3.4.0
- Notes
- -----
- The function is non-deterministic because its results depends on the order of the
- rows which may be non-deterministic after a shuffle.
+def last(col: "ColumnOrName", ignorenulls: bool = False) -> Column:
+ return _invoke_function("last", _to_col(col), lit(ignorenulls))
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- column to fetch last value for.
- ignorenulls : :class:`~pyspark.sql.Column` or str
- if last value is null then look for non-null value.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- last value of the group.
-
- Examples
- --------
- >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5), ("Alice", None)], ("name", "age"))
- >>> df = df.orderBy(df.age.desc())
- >>> df.groupby("name").agg(last("age")).orderBy("name").show()
- +-----+---------+
- | name|last(age)|
- +-----+---------+
- |Alice| null|
- | Bob| 5|
- +-----+---------+
-
- Now, to ignore any nulls we needs to set ``ignorenulls`` to `True`
-
- >>> df.groupby("name").agg(last("age", ignorenulls=True)).orderBy("name").show()
- +-----+---------+
- | name|last(age)|
- +-----+---------+
- |Alice| 2|
- | Bob| 5|
- +-----+---------+
- """
- return _invoke_function("last", _to_col(col), lit(ignorenulls))
+last.__doc__ = pysparkfuncs.last.__doc__
def max(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns the maximum value of the expression in a group.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("max", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- column for computed results.
-
- Examples
- --------
- >>> df = spark.range(10)
- >>> df.select(max(col("id"))).show()
- +-------+
- |max(id)|
- +-------+
- | 9|
- +-------+
- """
- return _invoke_function_over_columns("max", col)
+max.__doc__ = pysparkfuncs.max.__doc__
def max_by(col: "ColumnOrName", ord: "ColumnOrName") -> Column:
- """
- Returns the value associated with the maximum value of ord.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("max_by", col, ord)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- ord : :class:`~pyspark.sql.Column` or str
- column to be maximized
- Returns
- -------
- :class:`~pyspark.sql.Column`
- value associated with the maximum value of ord.
-
- Examples
- --------
- >>> df = spark.createDataFrame([
- ... ("Java", 2012, 20000), ("dotNET", 2012, 5000),
- ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)],
- ... schema=("course", "year", "earnings"))
- >>> df.groupby("course").agg(max_by("year", "earnings")).show()
- +------+----------------------+
- |course|max_by(year, earnings)|
- +------+----------------------+
- | Java| 2013|
- |dotNET| 2013|
- +------+----------------------+
- """
- return _invoke_function_over_columns("max_by", col, ord)
+max_by.__doc__ = pysparkfuncs.max_by.__doc__
def mean(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns the average of the values in a group.
- An alias of :func:`avg`.
-
- .. versionadded:: 1.4.0
+ return avg(col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.range(10)
- >>> df.select(mean(df.id)).show()
- +-------+
- |avg(id)|
- +-------+
- | 4.5|
- +-------+
- """
- return avg(col)
+mean.__doc__ = pysparkfuncs.mean.__doc__
def median(col: "ColumnOrName") -> Column:
- """
- Returns the median of the values in a group.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("median", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the median of the values in a group.
-
- Examples
- --------
- >>> df = spark.createDataFrame([
- ... ("Java", 2012, 20000), ("dotNET", 2012, 5000),
- ... ("Java", 2012, 22000), ("dotNET", 2012, 10000),
- ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)],
- ... schema=("course", "year", "earnings"))
- >>> df.groupby("course").agg(median("earnings")).show()
- +------+----------------+
- |course|median(earnings)|
- +------+----------------+
- | Java| 22000.0|
- |dotNET| 10000.0|
- +------+----------------+
- """
- return _invoke_function_over_columns("median", col)
+median.__doc__ = pysparkfuncs.median.__doc__
def min(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns the minimum value of the expression in a group.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("min", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- column for computed results.
-
- Examples
- --------
- >>> df = spark.range(10)
- >>> df.select(min(df.id)).show()
- +-------+
- |min(id)|
- +-------+
- | 0|
- +-------+
- """
- return _invoke_function_over_columns("min", col)
+min.__doc__ = pysparkfuncs.min.__doc__
def min_by(col: "ColumnOrName", ord: "ColumnOrName") -> Column:
- """
- Returns the value associated with the minimum value of ord.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("min_by", col, ord)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- ord : :class:`~pyspark.sql.Column` or str
- column to be minimized
- Returns
- -------
- :class:`~pyspark.sql.Column`
- value associated with the minimum value of ord.
-
- Examples
- --------
- >>> df = spark.createDataFrame([
- ... ("Java", 2012, 20000), ("dotNET", 2012, 5000),
- ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)],
- ... schema=("course", "year", "earnings"))
- >>> df.groupby("course").agg(min_by("year", "earnings")).show()
- +------+----------------------+
- |course|min_by(year, earnings)|
- +------+----------------------+
- | Java| 2012|
- |dotNET| 2012|
- +------+----------------------+
- """
- return _invoke_function_over_columns("min_by", col, ord)
+min_by.__doc__ = pysparkfuncs.min_by.__doc__
def mode(col: "ColumnOrName") -> Column:
- """
- Returns the most frequent value in a group.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("mode", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the most frequent value in a group.
-
- Examples
- --------
- >>> df = spark.createDataFrame([
- ... ("Java", 2012, 20000), ("dotNET", 2012, 5000),
- ... ("Java", 2012, 20000), ("dotNET", 2012, 5000),
- ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)],
- ... schema=("course", "year", "earnings"))
- >>> df.groupby("course").agg(mode("year")).show()
- +------+----------+
- |course|mode(year)|
- +------+----------+
- | Java| 2012|
- |dotNET| 2012|
- +------+----------+
- """
- return _invoke_function_over_columns("mode", col)
+mode.__doc__ = pysparkfuncs.mode.__doc__
def percentile_approx(
@@ -2941,52 +877,6 @@ def percentile_approx(
percentage: Union[Column, float, List[float], Tuple[float]],
accuracy: Union[Column, float] = 10000,
) -> Column:
- """Returns the approximate `percentile` of the numeric column `col` which is the smallest value
- in the ordered `col` values (sorted from least to greatest) such that no more than `percentage`
- of `col` values is less than the value or equal to that value.
-
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- input column.
- percentage : :class:`~pyspark.sql.Column`, float, list of floats or tuple of floats
- percentage in decimal (must be between 0.0 and 1.0).
- When percentage is an array, each value of the percentage array must be between 0.0 and 1.0.
- In this case, returns the approximate percentile array of column col
- at the given percentage array.
- accuracy : :class:`~pyspark.sql.Column` or float
- is a positive numeric literal which controls approximation accuracy
- at the cost of memory. Higher value of accuracy yields better accuracy,
- 1.0/accuracy is the relative error of the approximation. (default: 10000).
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- approximate `percentile` of the numeric column.
-
- Examples
- --------
- >>> key = (col("id") % 3).alias("key")
- >>> value = (randn(42) + key * 10).alias("value")
- >>> df = spark.range(0, 1000, 1, 1).select(key, value)
- >>> df.select(
- ... percentile_approx("value", [0.25, 0.5, 0.75], 1000000).alias("quantiles")
- ... ).printSchema()
- root
- |-- quantiles: array (nullable = true)
- | |-- element: double (containsNull = false)
-
- >>> df.groupBy("key").agg(
- ... percentile_approx("value", 0.5, lit(1000000)).alias("median")
- ... ).printSchema()
- root
- |-- key: long (nullable = true)
- |-- median: double (nullable = true)
- """
-
if isinstance(percentage, Column):
percentage_col = percentage
elif isinstance(percentage, (list, tuple)):
@@ -2999,755 +889,160 @@ def percentile_approx(
return _invoke_function("percentile_approx", _to_col(col), percentage_col, lit(accuracy))
-def product(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns the product of the values in a group.
-
- .. versionadded:: 3.4.0
+percentile_approx.__doc__ = pysparkfuncs.percentile_approx.__doc__
- Parameters
- ----------
- col : str, :class:`Column`
- column containing values to be multiplied together
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.range(1, 10).toDF('x').withColumn('mod3', col('x') % 3)
- >>> prods = df.groupBy('mod3').agg(product('x').alias('product'))
- >>> prods.orderBy('mod3').show()
- +----+-------+
- |mod3|product|
- +----+-------+
- | 0| 162.0|
- | 1| 28.0|
- | 2| 80.0|
- +----+-------+
- """
+def product(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("product", col)
-def skewness(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns the skewness of the values in a group.
-
- .. versionadded:: 3.4.0
+product.__doc__ = pysparkfuncs.product.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- skewness of given column.
-
- Examples
- --------
- >>> df = spark.createDataFrame([[1],[1],[2]], ["c"])
- >>> df.select(skewness(df.c)).first()
- Row(skewness(c)=0.70710...)
- """
+def skewness(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("skewness", col)
-def stddev(col: "ColumnOrName") -> Column:
- """
- Aggregate function: alias for stddev_samp.
-
- .. versionadded:: 3.4.0
+skewness.__doc__ = pysparkfuncs.skewness.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- standard deviation of given column.
-
- Examples
- --------
- >>> df = spark.range(6)
- >>> df.select(stddev(df.id)).first()
- Row(stddev_samp(id)=1.87082...)
- """
+def stddev(col: "ColumnOrName") -> Column:
return stddev_samp(col)
-def stddev_samp(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns the unbiased sample standard deviation of
- the expression in a group.
+stddev.__doc__ = pysparkfuncs.stddev.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- standard deviation of given column.
-
- Examples
- --------
- >>> df = spark.range(6)
- >>> df.select(stddev_samp(df.id)).first()
- Row(stddev_samp(id)=1.87082...)
- """
+def stddev_samp(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("stddev_samp", col)
-def stddev_pop(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns population standard deviation of
- the expression in a group.
-
- .. versionadded:: 3.4.0
+stddev_samp.__doc__ = pysparkfuncs.stddev_samp.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- standard deviation of given column.
-
- Examples
- --------
- >>> df = spark.range(6)
- >>> df.select(stddev_pop(df.id)).first()
- Row(stddev_pop(id)=1.70782...)
- """
+def stddev_pop(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("stddev_pop", col)
-def sum(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns the sum of all values in the expression.
-
- .. versionadded:: 3.4.0
+stddev_pop.__doc__ = pysparkfuncs.stddev_pop.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.range(10)
- >>> df.select(sum(df["id"])).show()
- +-------+
- |sum(id)|
- +-------+
- | 45|
- +-------+
- """
+def sum(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("sum", col)
-def sumDistinct(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns the sum of distinct values in the expression.
+sum.__doc__ = pysparkfuncs.sum.__doc__
- .. versionadded:: 3.4.0
- .. deprecated:: 3.4.0
- Use :func:`sum_distinct` instead.
- """
+def sumDistinct(col: "ColumnOrName") -> Column:
warnings.warn("Deprecated in 3.4, use sum_distinct instead.", FutureWarning)
return sum_distinct(col)
-def sum_distinct(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns the sum of distinct values in the expression.
-
- .. versionadded:: 3.4.0
+sumDistinct.__doc__ = pysparkfuncs.sumDistinct.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(None,), (1,), (1,), (2,)], schema=["numbers"])
- >>> df.select(sum_distinct(col("numbers"))).show()
- +---------------------+
- |sum(DISTINCT numbers)|
- +---------------------+
- | 3|
- +---------------------+
- """
+def sum_distinct(col: "ColumnOrName") -> Column:
return Column(UnresolvedFunction("sum", [_to_col(col)._expr], is_distinct=True))
-def var_pop(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns the population variance of the values in a group.
+sum_distinct.__doc__ = pysparkfuncs.sum_distinct.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- variance of given column.
-
- Examples
- --------
- >>> df = spark.range(6)
- >>> df.select(var_pop(df.id)).first()
- Row(var_pop(id)=2.91666...)
- """
+def var_pop(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("var_pop", col)
-def var_samp(col: "ColumnOrName") -> Column:
- """
- Aggregate function: returns the unbiased sample variance of
- the values in a group.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- variance of given column.
-
- Examples
- --------
- >>> df = spark.range(6)
- >>> df.select(var_samp(df.id)).show()
- +------------+
- |var_samp(id)|
- +------------+
- | 3.5|
- +------------+
- """
- return _invoke_function_over_columns("var_samp", col)
-
-
-def variance(col: "ColumnOrName") -> Column:
- """
- Aggregate function: alias for var_samp
-
- .. versionadded:: 3.4.0
+var_pop.__doc__ = pysparkfuncs.var_pop.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- variance of given column.
-
- Examples
- --------
- >>> df = spark.range(6)
- >>> df.select(variance(df.id)).show()
- +------------+
- |var_samp(id)|
- +------------+
- | 3.5|
- +------------+
- """
- return var_samp(col)
+def var_samp(col: "ColumnOrName") -> Column:
+ return _invoke_function_over_columns("var_samp", col)
-# Window Functions
+var_samp.__doc__ = pysparkfuncs.var_samp.__doc__
-def cume_dist() -> Column:
- """
- Window function: returns the cumulative distribution of values within a window partition,
- i.e. the fraction of rows that are below the current row.
+def variance(col: "ColumnOrName") -> Column:
+ return var_samp(col)
- .. versionadded:: 3.4.0
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for calculating cumulative distribution.
-
- Examples
- --------
- >>> from pyspark.sql import Window, types
- >>> df = spark.createDataFrame([1, 2, 3, 3, 4], types.IntegerType())
- >>> w = Window.orderBy("value")
- >>> df.withColumn("cd", cume_dist().over(w)).show()
- +-----+---+
- |value| cd|
- +-----+---+
- | 1|0.2|
- | 2|0.4|
- | 3|0.8|
- | 3|0.8|
- | 4|1.0|
- +-----+---+
- """
- return _invoke_function("cume_dist")
+variance.__doc__ = pysparkfuncs.variance.__doc__
-def dense_rank() -> Column:
- """
- Window function: returns the rank of rows within a window partition, without any gaps.
+# Window Functions
- The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking
- sequence when there are ties. That is, if you were ranking a competition using dense_rank
- and had three people tie for second place, you would say that all three were in second
- place and that the next person came in third. Rank would give me sequential numbers, making
- the person that came in third place (after the ties) would register as coming in fifth.
- This is equivalent to the DENSE_RANK function in SQL.
+def cume_dist() -> Column:
+ return _invoke_function("cume_dist")
- .. versionadded:: 3.4.0
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for calculating ranks.
-
- Examples
- --------
- >>> from pyspark.sql import Window, types
- >>> df = spark.createDataFrame([1, 1, 2, 3, 3, 4], types.IntegerType())
- >>> w = Window.orderBy("value")
- >>> df.withColumn("drank", dense_rank().over(w)).show()
- +-----+-----+
- |value|drank|
- +-----+-----+
- | 1| 1|
- | 1| 1|
- | 2| 2|
- | 3| 3|
- | 3| 3|
- | 4| 4|
- +-----+-----+
- """
- return _invoke_function("dense_rank")
+cume_dist.__doc__ = pysparkfuncs.cume_dist.__doc__
-def lag(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Column:
- """
- Window function: returns the value that is `offset` rows before the current row, and
- `default` if there is less than `offset` rows before the current row. For example,
- an `offset` of one will return the previous row at any given point in the window partition.
+def dense_rank() -> Column:
+ return _invoke_function("dense_rank")
- This is equivalent to the LAG function in SQL.
- .. versionadded:: 3.4.0
+dense_rank.__doc__ = pysparkfuncs.dense_rank.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- offset : int, optional default 1
- number of row to extend
- default : optional
- default value
- Returns
- -------
- :class:`~pyspark.sql.Column`
- value before current row based on `offset`.
-
- Examples
- --------
- >>> from pyspark.sql import Window
- >>> df = spark.createDataFrame([("a", 1),
- ... ("a", 2),
- ... ("a", 3),
- ... ("b", 8),
- ... ("b", 2)], ["c1", "c2"])
- >>> df.show()
- +---+---+
- | c1| c2|
- +---+---+
- | a| 1|
- | a| 2|
- | a| 3|
- | b| 8|
- | b| 2|
- +---+---+
- >>> w = Window.partitionBy("c1").orderBy("c2")
- >>> df.withColumn("previos_value", lag("c2").over(w)).show()
- +---+---+-------------+
- | c1| c2|previos_value|
- +---+---+-------------+
- | a| 1| null|
- | a| 2| 1|
- | a| 3| 2|
- | b| 2| null|
- | b| 8| 2|
- +---+---+-------------+
- >>> df.withColumn("previos_value", lag("c2", 1, 0).over(w)).show()
- +---+---+-------------+
- | c1| c2|previos_value|
- +---+---+-------------+
- | a| 1| 0|
- | a| 2| 1|
- | a| 3| 2|
- | b| 2| 0|
- | b| 8| 2|
- +---+---+-------------+
- >>> df.withColumn("previos_value", lag("c2", 2, -1).over(w)).show()
- +---+---+-------------+
- | c1| c2|previos_value|
- +---+---+-------------+
- | a| 1| -1|
- | a| 2| -1|
- | a| 3| 1|
- | b| 2| -1|
- | b| 8| -1|
- +---+---+-------------+
- """
+def lag(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Column:
if default is None:
return _invoke_function("lag", _to_col(col), lit(offset))
else:
return _invoke_function("lag", _to_col(col), lit(offset), lit(default))
-def lead(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Column:
- """
- Window function: returns the value that is `offset` rows after the current row, and
- `default` if there is less than `offset` rows after the current row. For example,
- an `offset` of one will return the next row at any given point in the window partition.
-
- This is equivalent to the LEAD function in SQL.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- offset : int, optional default 1
- number of row to extend
- default : optional
- default value
+lag.__doc__ = pysparkfuncs.lag.__doc__
- Returns
- -------
- :class:`~pyspark.sql.Column`
- value after current row based on `offset`.
-
- Examples
- --------
- >>> from pyspark.sql import Window
- >>> df = spark.createDataFrame([("a", 1),
- ... ("a", 2),
- ... ("a", 3),
- ... ("b", 8),
- ... ("b", 2)], ["c1", "c2"])
- >>> df.show()
- +---+---+
- | c1| c2|
- +---+---+
- | a| 1|
- | a| 2|
- | a| 3|
- | b| 8|
- | b| 2|
- +---+---+
- >>> w = Window.partitionBy("c1").orderBy("c2")
- >>> df.withColumn("next_value", lead("c2").over(w)).show()
- +---+---+----------+
- | c1| c2|next_value|
- +---+---+----------+
- | a| 1| 2|
- | a| 2| 3|
- | a| 3| null|
- | b| 2| 8|
- | b| 8| null|
- +---+---+----------+
- >>> df.withColumn("next_value", lead("c2", 1, 0).over(w)).show()
- +---+---+----------+
- | c1| c2|next_value|
- +---+---+----------+
- | a| 1| 2|
- | a| 2| 3|
- | a| 3| 0|
- | b| 2| 8|
- | b| 8| 0|
- +---+---+----------+
- >>> df.withColumn("next_value", lead("c2", 2, -1).over(w)).show()
- +---+---+----------+
- | c1| c2|next_value|
- +---+---+----------+
- | a| 1| 3|
- | a| 2| -1|
- | a| 3| -1|
- | b| 2| -1|
- | b| 8| -1|
- +---+---+----------+
- """
+def lead(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Column:
if default is None:
return _invoke_function("lead", _to_col(col), lit(offset))
else:
return _invoke_function("lead", _to_col(col), lit(offset), lit(default))
-def nth_value(col: "ColumnOrName", offset: int, ignoreNulls: Optional[bool] = None) -> Column:
- """
- Window function: returns the value that is the `offset`\\th row of the window frame
- (counting from 1), and `null` if the size of window frame is less than `offset` rows.
-
- It will return the `offset`\\th non-null value it sees when `ignoreNulls` is set to
- true. If all values are null, then null is returned.
-
- This is equivalent to the nth_value function in SQL.
+lead.__doc__ = pysparkfuncs.lead.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- offset : int
- number of row to use as the value
- ignoreNulls : bool, optional
- indicates the Nth value should skip null in the
- determination of which row to use
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- value of nth row.
-
- Examples
- --------
- >>> from pyspark.sql import Window
- >>> df = spark.createDataFrame([("a", 1),
- ... ("a", 2),
- ... ("a", 3),
- ... ("b", 8),
- ... ("b", 2)], ["c1", "c2"])
- >>> df.show()
- +---+---+
- | c1| c2|
- +---+---+
- | a| 1|
- | a| 2|
- | a| 3|
- | b| 8|
- | b| 2|
- +---+---+
- >>> w = Window.partitionBy("c1").orderBy("c2")
- >>> df.withColumn("nth_value", nth_value("c2", 1).over(w)).show()
- +---+---+---------+
- | c1| c2|nth_value|
- +---+---+---------+
- | a| 1| 1|
- | a| 2| 1|
- | a| 3| 1|
- | b| 2| 2|
- | b| 8| 2|
- +---+---+---------+
- >>> df.withColumn("nth_value", nth_value("c2", 2).over(w)).show()
- +---+---+---------+
- | c1| c2|nth_value|
- +---+---+---------+
- | a| 1| null|
- | a| 2| 2|
- | a| 3| 2|
- | b| 2| null|
- | b| 8| 8|
- +---+---+---------+
- """
+def nth_value(col: "ColumnOrName", offset: int, ignoreNulls: Optional[bool] = None) -> Column:
if ignoreNulls is None:
return _invoke_function("nth_value", _to_col(col), lit(offset))
else:
return _invoke_function("nth_value", _to_col(col), lit(offset), lit(ignoreNulls))
-def ntile(n: int) -> Column:
- """
- Window function: returns the ntile group id (from 1 to `n` inclusive)
- in an ordered window partition. For example, if `n` is 4, the first
- quarter of the rows will get value 1, the second quarter will get 2,
- the third quarter will get 3, and the last quarter will get 4.
-
- This is equivalent to the NTILE function in SQL.
+nth_value.__doc__ = pysparkfuncs.nth_value.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- n : int
- an integer
- Returns
- -------
- :class:`~pyspark.sql.Column`
- portioned group id.
-
- Examples
- --------
- >>> from pyspark.sql import Window
- >>> df = spark.createDataFrame([("a", 1),
- ... ("a", 2),
- ... ("a", 3),
- ... ("b", 8),
- ... ("b", 2)], ["c1", "c2"])
- >>> df.show()
- +---+---+
- | c1| c2|
- +---+---+
- | a| 1|
- | a| 2|
- | a| 3|
- | b| 8|
- | b| 2|
- +---+---+
- >>> w = Window.partitionBy("c1").orderBy("c2")
- >>> df.withColumn("ntile", ntile(2).over(w)).show()
- +---+---+-----+
- | c1| c2|ntile|
- +---+---+-----+
- | a| 1| 1|
- | a| 2| 1|
- | a| 3| 2|
- | b| 2| 1|
- | b| 8| 2|
- +---+---+-----+
- """
+def ntile(n: int) -> Column:
return _invoke_function("ntile", lit(n))
-def percent_rank() -> Column:
- """
- Window function: returns the relative rank (i.e. percentile) of rows within a window partition.
+ntile.__doc__ = pysparkfuncs.ntile.__doc__
- .. versionadded:: 3.4.0
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for calculating relative rank.
-
- Examples
- --------
- >>> from pyspark.sql import Window, types
- >>> df = spark.createDataFrame([1, 1, 2, 3, 3, 4], types.IntegerType())
- >>> w = Window.orderBy("value")
- >>> df.withColumn("pr", percent_rank().over(w)).show()
- +-----+---+
- |value| pr|
- +-----+---+
- | 1|0.0|
- | 1|0.0|
- | 2|0.4|
- | 3|0.6|
- | 3|0.6|
- | 4|1.0|
- +-----+---+
- """
+def percent_rank() -> Column:
return _invoke_function("percent_rank")
-def rank() -> Column:
- """
- Window function: returns the rank of rows within a window partition.
-
- The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking
- sequence when there are ties. That is, if you were ranking a competition using dense_rank
- and had three people tie for second place, you would say that all three were in second
- place and that the next person came in third. Rank would give me sequential numbers, making
- the person that came in third place (after the ties) would register as coming in fifth.
+percent_rank.__doc__ = pysparkfuncs.percent_rank.__doc__
- This is equivalent to the RANK function in SQL.
- .. versionadded:: 3.4.0
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for calculating ranks.
-
- Examples
- --------
- >>> from pyspark.sql import Window, types
- >>> df = spark.createDataFrame([1, 1, 2, 3, 3, 4], types.IntegerType())
- >>> w = Window.orderBy("value")
- >>> df.withColumn("drank", rank().over(w)).show()
- +-----+-----+
- |value|drank|
- +-----+-----+
- | 1| 1|
- | 1| 1|
- | 2| 3|
- | 3| 4|
- | 3| 4|
- | 4| 6|
- +-----+-----+
- """
+def rank() -> Column:
return _invoke_function("rank")
-def row_number() -> Column:
- """
- Window function: returns a sequential number starting at 1 within a window partition.
+rank.__doc__ = pysparkfuncs.rank.__doc__
- .. versionadded:: 3.4.0
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for calculating row numbers.
-
- Examples
- --------
- >>> from pyspark.sql import Window
- >>> df = spark.range(3)
- >>> w = Window.orderBy(df.id.desc())
- >>> df.withColumn("desc_order", row_number().over(w)).show()
- +---+----------+
- | id|desc_order|
- +---+----------+
- | 2| 1|
- | 1| 2|
- | 0| 3|
- +---+----------+
- """
+def row_number() -> Column:
return _invoke_function("row_number")
-# Collection Functions
+row_number.__doc__ = pysparkfuncs.row_number.__doc__
def aggregate(
@@ -3756,64 +1051,6 @@ def aggregate(
merge: Callable[[Column, Column], Column],
finish: Optional[Callable[[Column], Column]] = None,
) -> Column:
- """
- Applies a binary operator to an initial state and all elements in the array,
- and reduces this to a single state. The final state is converted into the final result
- by applying a finish function.
-
- Both functions can use methods of :class:`~pyspark.sql.Column`, functions defined in
- :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
- Python ``UserDefinedFunctions`` are not supported
- (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- initialValue : :class:`~pyspark.sql.Column` or str
- initial value. Name of column or expression
- merge : function
- a binary function ``(acc: Column, x: Column) -> Column...`` returning expression
- of the same type as ``zero``
- finish : function
- an optional unary function ``(x: Column) -> Column: ...``
- used to convert accumulated value.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- final value after aggregate function is applied.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(1, [20.0, 4.0, 2.0, 6.0, 10.0])], ("id", "values"))
- >>> df.select(aggregate("values", lit(0.0), lambda acc, x: acc + x).alias("sum")).show()
- +----+
- | sum|
- +----+
- |42.0|
- +----+
-
- >>> def merge(acc, x):
- ... count = acc.count + 1
- ... sum = acc.sum + x
- ... return struct(count.alias("count"), sum.alias("sum"))
- >>> df.select(
- ... aggregate(
- ... "values",
- ... struct(lit(0).alias("count"), lit(0.0).alias("sum")),
- ... merge,
- ... lambda acc: acc.sum / acc.count,
- ... ).alias("mean")
- ... ).show()
- +----+
- |mean|
- +----+
- | 8.4|
- +----+
- """
if finish is not None:
return _invoke_higher_order_function("aggregate", [col, initialValue], [merge, finish])
@@ -3821,828 +1058,195 @@ def aggregate(
return _invoke_higher_order_function("aggregate", [col, initialValue], [merge])
-def array(*cols: Union["ColumnOrName", List["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column:
- """Creates a new array column.
-
- .. versionadded:: 3.4.0
+aggregate.__doc__ = pysparkfuncs.aggregate.__doc__
- Parameters
- ----------
- cols : :class:`~pyspark.sql.Column` or str
- column names or :class:`~pyspark.sql.Column`\\s that have
- the same data type.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a column of array type.
-
- Examples
- --------
- >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5)], ("name", "age"))
- >>> df.select(array('age', 'age').alias("arr")).collect()
- [Row(arr=[2, 2]), Row(arr=[5, 5])]
- >>> df.select(array([df.age, df.age]).alias("arr")).collect()
- [Row(arr=[2, 2]), Row(arr=[5, 5])]
- >>> df.select(array('age', 'age').alias("col")).printSchema()
- root
- |-- col: array (nullable = false)
- | |-- element: long (containsNull = true)
- """
+def array(*cols: Union["ColumnOrName", List["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column:
if len(cols) == 1 and isinstance(cols[0], (list, set, tuple)):
cols = cols[0] # type: ignore[assignment]
return _invoke_function_over_columns("array", *cols) # type: ignore[arg-type]
-def array_contains(col: "ColumnOrName", value: Any) -> Column:
- """
- Collection function: returns null if the array is null, true if the array contains the
- given value, and false otherwise.
-
- .. versionadded:: 3.4.0
+array.__doc__ = pysparkfuncs.array.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column containing array
- value :
- value or column to check for in array
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a column of Boolean type.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(["a", "b", "c"],), ([],)], ['data'])
- >>> df.select(array_contains(df.data, "a")).collect()
- [Row(array_contains(data, a)=True), Row(array_contains(data, a)=False)]
- >>> df.select(array_contains(df.data, lit("a"))).collect()
- [Row(array_contains(data, a)=True), Row(array_contains(data, a)=False)]
- """
+def array_contains(col: "ColumnOrName", value: Any) -> Column:
return _invoke_function("array_contains", _to_col(col), lit(value))
-def array_distinct(col: "ColumnOrName") -> Column:
- """
- Collection function: removes duplicate values from the array.
+array_contains.__doc__ = pysparkfuncs.array_contains.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- Returns
- -------
- :class:`~pyspark.sql.Column`
- an array of unique values.
-
- Examples
- --------
- >>> df = spark.createDataFrame([([1, 2, 3, 2],), ([4, 5, 5, 4],)], ['data'])
- >>> df.select(array_distinct(df.data)).collect()
- [Row(array_distinct(data)=[1, 2, 3]), Row(array_distinct(data)=[4, 5])]
- """
+def array_distinct(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("array_distinct", col)
-def array_except(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
- """
- Collection function: returns an array of the elements in col1 but not in col2,
- without duplicates.
-
- .. versionadded:: 3.4.0
+array_distinct.__doc__ = pysparkfuncs.array_distinct.__doc__
- Parameters
- ----------
- col1 : :class:`~pyspark.sql.Column` or str
- name of column containing array
- col2 : :class:`~pyspark.sql.Column` or str
- name of column containing array
- Returns
- -------
- :class:`~pyspark.sql.Column`
- an array of values from first array that are not in the second.
-
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])])
- >>> df.select(array_except(df.c1, df.c2)).collect()
- [Row(array_except(c1, c2)=['b'])]
- """
+def array_except(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
return _invoke_function_over_columns("array_except", col1, col2)
-def array_intersect(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
- """
- Collection function: returns an array of the elements in the intersection of col1 and col2,
- without duplicates.
-
- .. versionadded:: 3.4.0
+array_except.__doc__ = pysparkfuncs.array_except.__doc__
- Parameters
- ----------
- col1 : :class:`~pyspark.sql.Column` or str
- name of column containing array
- col2 : :class:`~pyspark.sql.Column` or str
- name of column containing array
- Returns
- -------
- :class:`~pyspark.sql.Column`
- an array of values in the intersection of two arrays.
-
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])])
- >>> df.select(array_intersect(df.c1, df.c2)).collect()
- [Row(array_intersect(c1, c2)=['a', 'c'])]
- """
+def array_intersect(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
return _invoke_function_over_columns("array_intersect", col1, col2)
+array_intersect.__doc__ = pysparkfuncs.array_intersect.__doc__
+
+
def array_join(
col: "ColumnOrName", delimiter: str, null_replacement: Optional[str] = None
) -> Column:
- """
- Concatenates the elements of `column` using the `delimiter`. Null values are replaced with
- `null_replacement` if set, otherwise they are ignored.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- delimiter : str
- delimiter used to concatenate elements
- null_replacement : str, optional
- if set then null values will be replaced by this value
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a column of string type. Concatenated values.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(["a", "b", "c"],), (["a", None],)], ['data'])
- >>> df.select(array_join(df.data, ",").alias("joined")).collect()
- [Row(joined='a,b,c'), Row(joined='a')]
- >>> df.select(array_join(df.data, ",", "NULL").alias("joined")).collect()
- [Row(joined='a,b,c'), Row(joined='a,NULL')]
- """
if null_replacement is None:
return _invoke_function("array_join", _to_col(col), lit(delimiter))
else:
return _invoke_function("array_join", _to_col(col), lit(delimiter), lit(null_replacement))
-def array_max(col: "ColumnOrName") -> Column:
- """
- Collection function: returns the maximum value of the array.
-
- .. versionadded:: 3.4.0
+array_join.__doc__ = pysparkfuncs.array_join.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- Returns
- -------
- :class:`~pyspark.sql.Column`
- maximum value of an array.
-
- Examples
- --------
- >>> df = spark.createDataFrame([([2, 1, 3],), ([None, 10, -1],)], ['data'])
- >>> df.select(array_max(df.data).alias('max')).collect()
- [Row(max=3), Row(max=10)]
- """
+def array_max(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("array_max", col)
-def array_min(col: "ColumnOrName") -> Column:
- """
- Collection function: returns the minimum value of the array.
-
- .. versionadded:: 3.4.0
+array_max.__doc__ = pysparkfuncs.array_max.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- Returns
- -------
- :class:`~pyspark.sql.Column`
- minimum value of array.
-
- Examples
- --------
- >>> df = spark.createDataFrame([([2, 1, 3],), ([None, 10, -1],)], ['data'])
- >>> df.select(array_min(df.data).alias('min')).collect()
- [Row(min=1), Row(min=-1)]
- """
+def array_min(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("array_min", col)
-def array_position(col: "ColumnOrName", value: Any) -> Column:
- """
- Collection function: Locates the position of the first occurrence of the given value
- in the given array. Returns null if either of the arguments are null.
-
- .. versionadded:: 3.4.0
-
- Notes
- -----
- The position is not zero based, but 1 based index. Returns 0 if the given
- value could not be found in the array.
+array_min.__doc__ = pysparkfuncs.array_min.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- value : Any
- value to look for.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- position of the value in the given array if found and 0 otherwise.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(["c", "b", "a"],), ([],)], ['data'])
- >>> df.select(array_position(df.data, "a")).collect()
- [Row(array_position(data, a)=3), Row(array_position(data, a)=0)]
- """
+def array_position(col: "ColumnOrName", value: Any) -> Column:
return _invoke_function("array_position", _to_col(col), lit(value))
-def array_remove(col: "ColumnOrName", element: Any) -> Column:
- """
- Collection function: Remove all elements that equal to element from the given array.
-
- .. versionadded:: 3.4.0
+array_position.__doc__ = pysparkfuncs.array_position.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column containing array
- element :
- element to be removed from the array
- Returns
- -------
- :class:`~pyspark.sql.Column`
- an array excluding given value.
-
- Examples
- --------
- >>> df = spark.createDataFrame([([1, 2, 3, 1, 1],), ([],)], ['data'])
- >>> df.select(array_remove(df.data, 1)).collect()
- [Row(array_remove(data, 1)=[2, 3]), Row(array_remove(data, 1)=[])]
- """
+def array_remove(col: "ColumnOrName", element: Any) -> Column:
return _invoke_function("array_remove", _to_col(col), lit(element))
-def array_repeat(col: "ColumnOrName", count: Union["ColumnOrName", int]) -> Column:
- """
- Collection function: creates an array containing a column repeated count times.
-
- .. versionadded:: 3.4.0
+array_remove.__doc__ = pysparkfuncs.array_remove.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- column name or column that contains the element to be repeated
- count : :class:`~pyspark.sql.Column` or str or int
- column name, column, or int containing the number of times to repeat the first argument
- Returns
- -------
- :class:`~pyspark.sql.Column`
- an array of repeated elements.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('ab',)], ['data'])
- >>> df.select(array_repeat(df.data, 3).alias('r')).collect()
- [Row(r=['ab', 'ab', 'ab'])]
- """
+def array_repeat(col: "ColumnOrName", count: Union["ColumnOrName", int]) -> Column:
_count = lit(count) if isinstance(count, int) else _to_col(count)
-
return _invoke_function("array_repeat", _to_col(col), _count)
+array_repeat.__doc__ = pysparkfuncs.array_repeat.__doc__
+
+
def array_sort(
col: "ColumnOrName", comparator: Optional[Callable[[Column, Column], Column]] = None
) -> Column:
- """
- Collection function: sorts the input array in ascending order. The elements of the input array
- must be orderable. Null elements will be placed at the end of the returned array.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- comparator : callable, optional
- A binary ``(Column, Column) -> Column: ...``.
- The comparator will take two
- arguments representing two elements of the array. It returns a negative integer, 0, or a
- positive integer as the first element is less than, equal to, or greater than the second
- element. If the comparator function returns null, the function will fail and raise an
- error.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- sorted array.
-
- Examples
- --------
- >>> df = spark.createDataFrame([([2, 1, None, 3],),([1],),([],)], ['data'])
- >>> df.select(array_sort(df.data).alias('r')).collect()
- [Row(r=[1, 2, 3, None]), Row(r=[1]), Row(r=[])]
- >>> df = spark.createDataFrame([(["foo", "foobar", None, "bar"],),(["foo"],),([],)], ['data'])
- >>> df.select(array_sort(
- ... "data",
- ... lambda x, y: when(x.isNull() | y.isNull(), lit(0)).otherwise(length(y) - length(x))
- ... ).alias("r")).collect()
- [Row(r=['foobar', 'foo', None, 'bar']), Row(r=['foo']), Row(r=[])]
- """
if comparator is None:
return _invoke_function_over_columns("array_sort", col)
else:
return _invoke_higher_order_function("array_sort", [col], [comparator])
-def array_union(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
- """
- Collection function: returns an array of the elements in the union of col1 and col2,
- without duplicates.
+array_sort.__doc__ = pysparkfuncs.array_sort.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col1 : :class:`~pyspark.sql.Column` or str
- name of column containing array
- col2 : :class:`~pyspark.sql.Column` or str
- name of column containing array
- Returns
- -------
- :class:`~pyspark.sql.Column`
- an array of values in union of two arrays.
-
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])])
- >>> df.select(array_union(df.c1, df.c2)).collect()
- [Row(array_union(c1, c2)=['b', 'a', 'c', 'd', 'f'])]
- """
+def array_union(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
return _invoke_function_over_columns("array_union", col1, col2)
-def arrays_overlap(a1: "ColumnOrName", a2: "ColumnOrName") -> Column:
- """
- Collection function: returns true if the arrays contain any common non-null element; if not,
- returns null if both the arrays are non-empty and any of them contains a null element; returns
- false otherwise.
+array_union.__doc__ = pysparkfuncs.array_union.__doc__
- .. versionadded:: 3.4.0
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a column of Boolean type.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(["a", "b"], ["b", "c"]), (["a"], ["b", "c"])], ['x', 'y'])
- >>> df.select(arrays_overlap(df.x, df.y).alias("overlap")).collect()
- [Row(overlap=True), Row(overlap=False)]
- """
+def arrays_overlap(a1: "ColumnOrName", a2: "ColumnOrName") -> Column:
return _invoke_function_over_columns("arrays_overlap", a1, a2)
-def arrays_zip(*cols: "ColumnOrName") -> Column:
- """
- Collection function: Returns a merged array of structs in which the N-th struct contains all
- N-th values of input arrays. If one of the arrays is shorter than others then
- resulting struct type value will be a `null` for missing elements.
-
- .. versionadded:: 2.4.0
+arrays_overlap.__doc__ = pysparkfuncs.arrays_overlap.__doc__
- Parameters
- ----------
- cols : :class:`~pyspark.sql.Column` or str
- columns of arrays to be merged.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- merged array of entries.
-
- Examples
- --------
- >>> from pyspark.sql.functions import arrays_zip
- >>> df = spark.createDataFrame([(([1, 2, 3], [2, 4, 6], [3, 6]))], ['vals1', 'vals2', 'vals3'])
- >>> df = df.select(arrays_zip(df.vals1, df.vals2, df.vals3).alias('zipped'))
- >>> df.show(truncate=False)
- +------------------------------------+
- |zipped |
- +------------------------------------+
- |[{1, 2, 3}, {2, 4, 6}, {3, 6, null}]|
- +------------------------------------+
- >>> df.printSchema()
- root
- |-- zipped: array (nullable = true)
- | |-- element: struct (containsNull = false)
- | | |-- vals1: long (nullable = true)
- | | |-- vals2: long (nullable = true)
- | | |-- vals3: long (nullable = true)
- """
+def arrays_zip(*cols: "ColumnOrName") -> Column:
return _invoke_function_over_columns("arrays_zip", *cols)
-def concat(*cols: "ColumnOrName") -> Column:
- """
- Concatenates multiple input columns together into a single column.
- The function works with strings, numeric, binary and compatible array columns.
+arrays_zip.__doc__ = pysparkfuncs.arrays_zip.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- cols : :class:`~pyspark.sql.Column` or str
- target column or columns to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- concatenated values. Type of the `Column` depends on input columns' type.
-
- See Also
- --------
- :meth:`pyspark.sql.functions.array_join` : to concatenate string columns with delimiter
-
- Examples
- --------
- >>> df = spark.createDataFrame([('abcd','123')], ['s', 'd'])
- >>> df = df.select(concat(df.s, df.d).alias('s'))
- >>> df.collect()
- [Row(s='abcd123')]
- >>> df
- DataFrame[s: string]
-
- >>> df = spark.createDataFrame([([1, 2], [3, 4], [5]), ([1, 2], None, [3])], ['a', 'b', 'c'])
- >>> df = df.select(concat(df.a, df.b, df.c).alias("arr"))
- >>> df.collect()
- [Row(arr=[1, 2, 3, 4, 5]), Row(arr=None)]
- >>> df
- DataFrame[arr: array<bigint>]
- """
+def concat(*cols: "ColumnOrName") -> Column:
return _invoke_function_over_columns("concat", *cols)
+concat.__doc__ = pysparkfuncs.concat.__doc__
+
+
def create_map(
*cols: Union["ColumnOrName", List["ColumnOrName"], Tuple["ColumnOrName", ...]]
) -> Column:
- """Creates a new map column.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- cols : :class:`~pyspark.sql.Column` or str
- column names or :class:`~pyspark.sql.Column`\\s that are
- grouped as key-value pairs, e.g. (key1, value1, key2, value2, ...).
-
- Examples
- --------
- >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5)], ("name", "age"))
- >>> df.select(create_map('name', 'age').alias("map")).collect()
- [Row(map={'Alice': 2}), Row(map={'Bob': 5})]
- >>> df.select(create_map([df.name, df.age]).alias("map")).collect()
- [Row(map={'Alice': 2}), Row(map={'Bob': 5})]
- """
if len(cols) == 1 and isinstance(cols[0], (list, set, tuple)):
cols = cols[0] # type: ignore[assignment]
return _invoke_function_over_columns("map", *cols) # type: ignore[arg-type]
-def element_at(col: "ColumnOrName", extraction: Any) -> Column:
- """
- Collection function: Returns element of array at given index in `extraction` if col is array.
- Returns value for the given key in `extraction` if col is map. If position is negative
- then location of the element will start from end, if number is outside the
- array boundaries then None will be returned.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column containing array or map
- extraction :
- index to check for in array or key to check for in map
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- value at given position.
-
- Notes
- -----
- The position is not zero based, but 1 based index.
-
- See Also
- --------
- :meth:`get`
-
- Examples
- --------
- >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data'])
- >>> df.select(element_at(df.data, 1)).collect()
- [Row(element_at(data, 1)='a')]
- >>> df.select(element_at(df.data, -1)).collect()
- [Row(element_at(data, -1)='c')]
-
- >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data'])
- >>> df.select(element_at(df.data, lit("a"))).collect()
- [Row(element_at(data, a)=1.0)]
- """
- return _invoke_function("element_at", _to_col(col), lit(extraction))
-
+create_map.__doc__ = pysparkfuncs.create_map.__doc__
-def exists(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column:
- """
- Returns whether a predicate holds for one or more elements in the array.
- .. versionadded:: 3.4.0
+def element_at(col: "ColumnOrName", extraction: Any) -> Column:
+ return _invoke_function("element_at", _to_col(col), lit(extraction))
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- f : function
- ``(x: Column) -> Column: ...`` returning the Boolean expression.
- Can use methods of :class:`~pyspark.sql.Column`, functions defined in
- :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
- Python ``UserDefinedFunctions`` are not supported
- (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
- Returns
- -------
- :class:`~pyspark.sql.Column`
- True if "any" element of an array evaluates to True when passed as an argument to
- given function and False otherwise.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(1, [1, 2, 3, 4]), (2, [3, -1, 0])],("key", "values"))
- >>> df.select(exists("values", lambda x: x < 0).alias("any_negative")).show()
- +------------+
- |any_negative|
- +------------+
- | false|
- | true|
- +------------+
- """
- return _invoke_higher_order_function("exists", [col], [f])
+element_at.__doc__ = pysparkfuncs.element_at.__doc__
-def explode(col: "ColumnOrName") -> Column:
- """
- Returns a new row for each element in the given array or map.
- Uses the default column name `col` for elements in the array and
- `key` and `value` for elements in the map unless specified otherwise.
+def exists(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column:
+ return _invoke_higher_order_function("exists", [col], [f])
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
+exists.__doc__ = pysparkfuncs.exists.__doc__
- Returns
- -------
- :class:`~pyspark.sql.Column`
- one row per array item or map key value.
-
- See Also
- --------
- :meth:`pyspark.functions.posexplode`
- :meth:`pyspark.functions.explode_outer`
- :meth:`pyspark.functions.posexplode_outer`
-
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> eDF = spark.createDataFrame([Row(a=1, intlist=[1,2,3], mapfield={"a": "b"})])
- >>> eDF.select(explode(eDF.intlist).alias("anInt")).collect()
- [Row(anInt=1), Row(anInt=2), Row(anInt=3)]
-
- >>> eDF.select(explode(eDF.mapfield).alias("key", "value")).show()
- +---+-----+
- |key|value|
- +---+-----+
- | a| b|
- +---+-----+
- """
- return _invoke_function_over_columns("explode", col)
+def explode(col: "ColumnOrName") -> Column:
+ return _invoke_function_over_columns("explode", col)
-def explode_outer(col: "ColumnOrName") -> Column:
- """
- Returns a new row for each element in the given array or map.
- Unlike explode, if the array/map is null or empty then null is produced.
- Uses the default column name `col` for elements in the array and
- `key` and `value` for elements in the map unless specified otherwise.
- .. versionadded:: 3.4.0
+explode.__doc__ = pysparkfuncs.explode.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- one row per array item or map key value.
-
- Examples
- --------
- >>> df = spark.createDataFrame(
- ... [(1, ["foo", "bar"], {"x": 1.0}), (2, [], {}), (3, None, None)],
- ... ("id", "an_array", "a_map")
- ... )
- >>> df.select("id", "an_array", explode_outer("a_map")).show()
- +---+----------+----+-----+
- | id| an_array| key|value|
- +---+----------+----+-----+
- | 1|[foo, bar]| x| 1.0|
- | 2| []|null| null|
- | 3| null|null| null|
- +---+----------+----+-----+
-
- >>> df.select("id", "a_map", explode_outer("an_array")).show()
- +---+----------+----+
- | id| a_map| col|
- +---+----------+----+
- | 1|{x -> 1.0}| foo|
- | 1|{x -> 1.0}| bar|
- | 2| {}|null|
- | 3| null|null|
- +---+----------+----+
- """
+def explode_outer(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("explode_outer", col)
+explode_outer.__doc__ = pysparkfuncs.explode_outer.__doc__
+
+
def filter(
col: "ColumnOrName",
f: Union[Callable[[Column], Column], Callable[[Column, Column], Column]],
) -> Column:
- """
- Returns an array of elements for which a predicate holds in a given array.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- f : function
- A function that returns the Boolean expression.
- Can take one of the following forms:
-
- - Unary ``(x: Column) -> Column: ...``
- - Binary ``(x: Column, i: Column) -> Column...``, where the second argument is
- a 0-based index of the element.
+ return _invoke_higher_order_function("filter", [col], [f])
- and can use methods of :class:`~pyspark.sql.Column`, functions defined in
- :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
- Python ``UserDefinedFunctions`` are not supported
- (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
- Returns
- -------
- :class:`~pyspark.sql.Column`
- filtered array of elements where given function evaluated to True
- when passed as an argument.
-
- Examples
- --------
- >>> df = spark.createDataFrame(
- ... [(1, ["2018-09-20", "2019-02-03", "2019-07-01", "2020-06-01"])],
- ... ("key", "values")
- ... )
- >>> def after_second_quarter(x):
- ... return month(to_date(x)) > 6
- >>> df.select(
- ... filter("values", after_second_quarter).alias("after_second_quarter")
- ... ).show(truncate=False)
- +------------------------+
- |after_second_quarter |
- +------------------------+
- |[2018-09-20, 2019-07-01]|
- +------------------------+
- """
- return _invoke_higher_order_function("filter", [col], [f])
+filter.__doc__ = pysparkfuncs.filter.__doc__
def flatten(col: "ColumnOrName") -> Column:
- """
- Collection function: creates a single array from an array of arrays.
- If a structure of nested arrays is deeper than two levels,
- only one level of nesting is removed.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("flatten", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- Returns
- -------
- :class:`~pyspark.sql.Column`
- flattened array.
-
- Examples
- --------
- >>> df = spark.createDataFrame([([[1, 2, 3], [4, 5], [6]],), ([None, [4, 5]],)], ['data'])
- >>> df.show(truncate=False)
- +------------------------+
- |data |
- +------------------------+
- |[[1, 2, 3], [4, 5], [6]]|
- |[null, [4, 5]] |
- +------------------------+
- >>> df.select(flatten(df.data).alias('r')).show()
- +------------------+
- | r|
- +------------------+
- |[1, 2, 3, 4, 5, 6]|
- | null|
- +------------------+
- """
- return _invoke_function_over_columns("flatten", col)
+flatten.__doc__ = pysparkfuncs.flatten.__doc__
def forall(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column:
- """
- Returns whether a predicate holds for every element in the array.
-
- .. versionadded:: 3.4.0
+ return _invoke_higher_order_function("forall", [col], [f])
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- f : function
- ``(x: Column) -> Column: ...`` returning the Boolean expression.
- Can use methods of :class:`~pyspark.sql.Column`, functions defined in
- :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
- Python ``UserDefinedFunctions`` are not supported
- (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
- Returns
- -------
- :class:`~pyspark.sql.Column`
- True if "all" elements of an array evaluates to True when passed as an argument to
- given function and False otherwise.
-
- Examples
- --------
- >>> df = spark.createDataFrame(
- ... [(1, ["bar"]), (2, ["foo", "bar"]), (3, ["foobar", "foo"])],
- ... ("key", "values")
- ... )
- >>> df.select(forall("values", lambda x: x.rlike("foo")).alias("all_foo")).show()
- +-------+
- |all_foo|
- +-------+
- | false|
- | false|
- | true|
- +-------+
- """
- return _invoke_higher_order_function("forall", [col], [f])
+forall.__doc__ = pysparkfuncs.forall.__doc__
# TODO: support options
@@ -4650,41 +1254,6 @@ def from_csv(
col: "ColumnOrName",
schema: Union[Column, str],
) -> Column:
- """
- Parses a column containing a CSV string to a row with the specified schema.
- Returns `null`, in the case of an unparseable string.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- a column or column name in CSV format
- schema :class:`~pyspark.sql.Column` or str
- a column, or Python string literal with schema in DDL format, to use
- when parsing the CSV column.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a column of parsed CSV values
-
- Examples
- --------
- >>> data = [("1,2,3",)]
- >>> df = spark.createDataFrame(data, ("value",))
- >>> df.select(from_csv(df.value, "a INT, b INT, c INT").alias("csv")).collect()
- [Row(csv=Row(a=1, b=2, c=3))]
- >>> value = data[0][0]
- >>> df.select(from_csv(df.value, schema_of_csv(value)).alias("csv")).collect()
- [Row(csv=Row(_c0=1, _c1=2, _c2=3))]
- >>> data = [(" abc",)]
- >>> df = spark.createDataFrame(data, ("value",))
- >>> options = {'ignoreLeadingWhiteSpace': True}
- >>> df.select(from_csv(df.value, "s string", options).alias("csv")).collect()
- [Row(csv=Row(s='abc'))]
- """
-
if isinstance(schema, Column):
_schema = schema
elif isinstance(schema, str):
@@ -4695,58 +1264,14 @@ def from_csv(
return _invoke_function("from_csv", _to_col(col), _schema)
+from_csv.__doc__ = pysparkfuncs.from_csv.__doc__
+
+
# TODO: 1, support ArrayType and StructType schema; 2, support options
def from_json(
col: "ColumnOrName",
schema: Union[Column, str],
) -> Column:
- """
- Parses a column containing a JSON string into a :class:`MapType` with :class:`StringType`
- as keys type, :class:`StructType` or :class:`ArrayType` with
- the specified schema. Returns `null`, in the case of an unparseable string.
-
- .. versionadded:: 2.1.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- a column or column name in JSON format
- schema :class:`~pyspark.sql.Column` or str
- a column, or Python string literal with schema in DDL format, to use when
- parsing the JSON column.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a new column of complex type from given JSON object.
-
- Examples
- --------
- >>> from pyspark.sql.types import *
- >>> data = [(1, '''{"a": 1}''')]
- >>> schema = StructType([StructField("a", IntegerType())])
- >>> df = spark.createDataFrame(data, ("key", "value"))
- >>> df.select(from_json(df.value, schema).alias("json")).collect()
- [Row(json=Row(a=1))]
- >>> df.select(from_json(df.value, "a INT").alias("json")).collect()
- [Row(json=Row(a=1))]
- >>> df.select(from_json(df.value, "MAP<STRING,INT>").alias("json")).collect()
- [Row(json={'a': 1})]
- >>> data = [(1, '''[{"a": 1}]''')]
- >>> schema = ArrayType(StructType([StructField("a", IntegerType())]))
- >>> df = spark.createDataFrame(data, ("key", "value"))
- >>> df.select(from_json(df.value, schema).alias("json")).collect()
- [Row(json=[Row(a=1)])]
- >>> schema = schema_of_json(lit('''{"a": 0}'''))
- >>> df.select(from_json(df.value, schema).alias("json")).collect()
- [Row(json=Row(a=None))]
- >>> data = [(1, '''[1, 2, 3]''')]
- >>> schema = ArrayType(IntegerType())
- >>> df = spark.createDataFrame(data, ("key", "value"))
- >>> df.select(from_json(df.value, schema).alias("json")).collect()
- [Row(json=[1, 2, 3])]
- """
-
if isinstance(schema, Column):
_schema = schema
elif isinstance(schema, str):
@@ -4757,665 +1282,140 @@ def from_json(
return _invoke_function("from_json", _to_col(col), _schema)
-def get(col: "ColumnOrName", index: Union["ColumnOrName", int]) -> Column:
- """
- Collection function: Returns element of array at given (0-based) index.
- If the index points outside of the array boundaries, then this function
- returns NULL.
-
- .. versionadded:: 3.4.0
+from_json.__doc__ = pysparkfuncs.from_json.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column containing array
- index : :class:`~pyspark.sql.Column` or str or int
- index to check for in array
- Returns
- -------
- :class:`~pyspark.sql.Column`
- value at given position.
-
- Notes
- -----
- The position is not 1 based, but 0 based index.
-
- See Also
- --------
- :meth:`element_at`
-
- Examples
- --------
- >>> df = spark.createDataFrame([(["a", "b", "c"], 1)], ['data', 'index'])
- >>> df.select(get(df.data, 1)).show()
- +------------+
- |get(data, 1)|
- +------------+
- | b|
- +------------+
-
- >>> df.select(get(df.data, -1)).show()
- +-------------+
- |get(data, -1)|
- +-------------+
- | null|
- +-------------+
-
- >>> df.select(get(df.data, 3)).show()
- +------------+
- |get(data, 3)|
- +------------+
- | null|
- +------------+
-
- >>> df.select(get(df.data, "index")).show()
- +----------------+
- |get(data, index)|
- +----------------+
- | b|
- +----------------+
-
- >>> df.select(get(df.data, col("index") - 1)).show()
- +----------------------+
- |get(data, (index - 1))|
- +----------------------+
- | a|
- +----------------------+
- """
+def get(col: "ColumnOrName", index: Union["ColumnOrName", int]) -> Column:
index = lit(index) if isinstance(index, int) else index
return _invoke_function_over_columns("get", col, index)
-def get_json_object(col: "ColumnOrName", path: str) -> Column:
- """
- Extracts json object from a json string based on json `path` specified, and returns json string
- of the extracted json object. It will return null if the input json string is invalid.
+get.__doc__ = pysparkfuncs.get.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- string column in json format
- path : str
- path to the json object to extract
- Returns
- -------
- :class:`~pyspark.sql.Column`
- string representation of given JSON object value.
-
- Examples
- --------
- >>> data = [("1", '''{"f1": "value1", "f2": "value2"}'''), ("2", '''{"f1": "value12"}''')]
- >>> df = spark.createDataFrame(data, ("key", "jstring"))
- >>> df.select(df.key, get_json_object(df.jstring, '$.f1').alias("c0"), \\
- ... get_json_object(df.jstring, '$.f2').alias("c1") ).collect()
- [Row(key='1', c0='value1', c1='value2'), Row(key='2', c0='value12', c1=None)]
- """
+def get_json_object(col: "ColumnOrName", path: str) -> Column:
return _invoke_function("get_json_object", _to_col(col), lit(path))
-def inline(col: "ColumnOrName") -> Column:
- """
- Explodes an array of structs into a table.
-
- .. versionadded:: 3.4.0
+get_json_object.__doc__ = pysparkfuncs.get_json_object.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- input column of values to explode.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- generator expression with the inline exploded result.
-
- See Also
- --------
- :meth:`explode`
-
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([Row(structlist=[Row(a=1, b=2), Row(a=3, b=4)])])
- >>> df.select(inline(df.structlist)).show()
- +---+---+
- | a| b|
- +---+---+
- | 1| 2|
- | 3| 4|
- +---+---+
- """
+def inline(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("inline", col)
-def inline_outer(col: "ColumnOrName") -> Column:
- """
- Explodes an array of structs into a table.
- Unlike inline, if the array is null or empty then null is produced for each nested column.
-
- .. versionadded:: 3.4.0
+inline.__doc__ = pysparkfuncs.inline.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- input column of values to explode.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- generator expression with the inline exploded result.
-
- See Also
- --------
- :meth:`explode_outer`
- :meth:`inline`
-
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([
- ... Row(id=1, structlist=[Row(a=1, b=2), Row(a=3, b=4)]),
- ... Row(id=2, structlist=[])
- ... ])
- >>> df.select('id', inline_outer(df.structlist)).show()
- +---+----+----+
- | id| a| b|
- +---+----+----+
- | 1| 1| 2|
- | 1| 3| 4|
- | 2|null|null|
- +---+----+----+
- """
+def inline_outer(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("inline_outer", col)
-def json_tuple(col: "ColumnOrName", *fields: str) -> Column:
- """Creates a new row for a json column according to the given field names.
+inline_outer.__doc__ = pysparkfuncs.inline_outer.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- string column in json format
- fields : str
- a field or fields to extract
+def json_tuple(col: "ColumnOrName", *fields: str) -> Column:
+ return _invoke_function("json_tuple", _to_col(col), *[lit(field) for field in fields])
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a new row for each given field value from json object
-
- Examples
- --------
- >>> data = [("1", '''{"f1": "value1", "f2": "value2"}'''), ("2", '''{"f1": "value12"}''')]
- >>> df = spark.createDataFrame(data, ("key", "jstring"))
- >>> df.select(df.key, json_tuple(df.jstring, 'f1', 'f2')).collect()
- [Row(key='1', c0='value1', c1='value2'), Row(key='2', c0='value12', c1=None)]
- """
- return _invoke_function("json_tuple", _to_col(col), *[lit(field) for field in fields])
+json_tuple.__doc__ = pysparkfuncs.json_tuple.__doc__
def map_concat(
*cols: Union["ColumnOrName", List["ColumnOrName"], Tuple["ColumnOrName", ...]]
) -> Column:
- """Returns the union of all the given maps.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- cols : :class:`~pyspark.sql.Column` or str
- column names or :class:`~pyspark.sql.Column`\\s
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a map of merged entries from other maps.
-
- Examples
- --------
- >>> from pyspark.sql.functions import map_concat
- >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(3, 'c') as map2")
- >>> df.select(map_concat("map1", "map2").alias("map3")).show(truncate=False)
- +------------------------+
- |map3 |
- +------------------------+
- |{1 -> a, 2 -> b, 3 -> c}|
- +------------------------+
- """
if len(cols) == 1 and isinstance(cols[0], (list, set, tuple)):
cols = cols[0] # type: ignore[assignment]
return _invoke_function_over_columns("map_concat", *cols) # type: ignore[arg-type]
-def map_contains_key(col: "ColumnOrName", value: Any) -> Column:
- """
- Returns true if the map contains the key.
+map_concat.__doc__ = pysparkfuncs.map_concat.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- value :
- a literal value
- Returns
- -------
- :class:`~pyspark.sql.Column`
- True if key is in the map and False otherwise.
-
- Examples
- --------
- >>> from pyspark.sql.functions import map_contains_key
- >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data")
- >>> df.select(map_contains_key("data", 1)).show()
- +---------------------------------+
- |array_contains(map_keys(data), 1)|
- +---------------------------------+
- | true|
- +---------------------------------+
- >>> df.select(map_contains_key("data", -1)).show()
- +----------------------------------+
- |array_contains(map_keys(data), -1)|
- +----------------------------------+
- | false|
- +----------------------------------+
- """
+def map_contains_key(col: "ColumnOrName", value: Any) -> Column:
return array_contains(map_keys(col), lit(value))
-def map_entries(col: "ColumnOrName") -> Column:
- """
- Collection function: Returns an unordered array of all entries in the given map.
-
- .. versionadded:: 3.4.0
+map_contains_key.__doc__ = pysparkfuncs.map_contains_key.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- Returns
- -------
- :class:`~pyspark.sql.Column`
- ar array of key value pairs as a struct type
-
- Examples
- --------
- >>> from pyspark.sql.functions import map_entries
- >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data")
- >>> df = df.select(map_entries("data").alias("entries"))
- >>> df.show()
- +----------------+
- | entries|
- +----------------+
- |[{1, a}, {2, b}]|
- +----------------+
- >>> df.printSchema()
- root
- |-- entries: array (nullable = false)
- | |-- element: struct (containsNull = false)
- | | |-- key: integer (nullable = false)
- | | |-- value: string (nullable = false)
- """
+def map_entries(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("map_entries", col)
-def map_filter(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column:
- """
- Returns a map whose key-value pairs satisfy a predicate.
-
- .. versionadded:: 3.4.0
+map_entries.__doc__ = pysparkfuncs.map_entries.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- f : function
- a binary function ``(k: Column, v: Column) -> Column...``
- Can use methods of :class:`~pyspark.sql.Column`, functions defined in
- :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
- Python ``UserDefinedFunctions`` are not supported
- (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
- Returns
- -------
- :class:`~pyspark.sql.Column`
- filtered map.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz": 32.0})], ("id", "data"))
- >>> df.select(map_filter(
- ... "data", lambda _, v: v > 30.0).alias("data_filtered")
- ... ).show(truncate=False)
- +--------------------------+
- |data_filtered |
- +--------------------------+
- |{baz -> 32.0, foo -> 42.0}|
- +--------------------------+
- """
+def map_filter(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column:
return _invoke_higher_order_function("map_filter", [col], [f])
-def map_from_arrays(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
- """Creates a new map from two arrays.
-
- .. versionadded:: 3.4.0
+map_filter.__doc__ = pysparkfuncs.map_filter.__doc__
- Parameters
- ----------
- col1 : :class:`~pyspark.sql.Column` or str
- name of column containing a set of keys. All elements should not be null
- col2 : :class:`~pyspark.sql.Column` or str
- name of column containing a set of values
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a column of map type.
-
- Examples
- --------
- >>> df = spark.createDataFrame([([2, 5], ['a', 'b'])], ['k', 'v'])
- >>> df = df.select(map_from_arrays(df.k, df.v).alias("col"))
- >>> df.show()
- +----------------+
- | col|
- +----------------+
- |{2 -> a, 5 -> b}|
- +----------------+
- >>> df.printSchema()
- root
- |-- col: map (nullable = true)
- | |-- key: long
- | |-- value: string (valueContainsNull = true)
- """
+def map_from_arrays(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
return _invoke_function_over_columns("map_from_arrays", col1, col2)
-def map_from_entries(col: "ColumnOrName") -> Column:
- """
- Collection function: Converts an array of entries (key value struct types) to a map
- of values.
-
- .. versionadded:: 3.4.0
+map_from_arrays.__doc__ = pysparkfuncs.map_from_arrays.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a map created from the given array of entries.
-
- Examples
- --------
- >>> from pyspark.sql.functions import map_from_entries
- >>> df = spark.sql("SELECT array(struct(1, 'a'), struct(2, 'b')) as data")
- >>> df.select(map_from_entries("data").alias("map")).show()
- +----------------+
- | map|
- +----------------+
- |{1 -> a, 2 -> b}|
- +----------------+
- """
+def map_from_entries(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("map_from_entries", col)
-def map_keys(col: "ColumnOrName") -> Column:
- """
- Collection function: Returns an unordered array containing the keys of the map.
+map_from_entries.__doc__ = pysparkfuncs.map_from_entries.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- Returns
- -------
- :class:`~pyspark.sql.Column`
- keys of the map as an array.
-
- Examples
- --------
- >>> from pyspark.sql.functions import map_keys
- >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data")
- >>> df.select(map_keys("data").alias("keys")).show()
- +------+
- | keys|
- +------+
- |[1, 2]|
- +------+
- """
+def map_keys(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("map_keys", col)
-def map_values(col: "ColumnOrName") -> Column:
- """
- Collection function: Returns an unordered array containing the values of the map.
-
- .. versionadded:: 3.4.0
+map_keys.__doc__ = pysparkfuncs.map_keys.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- Returns
- -------
- :class:`~pyspark.sql.Column`
- values of the map as an array.
-
- Examples
- --------
- >>> from pyspark.sql.functions import map_values
- >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data")
- >>> df.select(map_values("data").alias("values")).show()
- +------+
- |values|
- +------+
- |[a, b]|
- +------+
- """
+def map_values(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("map_values", col)
+map_values.__doc__ = pysparkfuncs.map_values.__doc__
+
+
def map_zip_with(
col1: "ColumnOrName",
col2: "ColumnOrName",
f: Callable[[Column, Column, Column], Column],
) -> Column:
- """
- Merge two given maps, key-wise into a single map using a function.
-
- .. versionadded:: 3.4.0
+ return _invoke_higher_order_function("map_zip_with", [col1, col2], [f])
- Parameters
- ----------
- col1 : :class:`~pyspark.sql.Column` or str
- name of the first column or expression
- col2 : :class:`~pyspark.sql.Column` or str
- name of the second column or expression
- f : function
- a ternary function ``(k: Column, v1: Column, v2: Column) -> Column...``
- Can use methods of :class:`~pyspark.sql.Column`, functions defined in
- :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
- Python ``UserDefinedFunctions`` are not supported
- (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
- Returns
- -------
- :class:`~pyspark.sql.Column`
- zipped map where entries are calculated by applying given function to each
- pair of arguments.
-
- Examples
- --------
- >>> df = spark.createDataFrame([
- ... (1, {"IT": 24.0, "SALES": 12.00}, {"IT": 2.0, "SALES": 1.4})],
- ... ("id", "base", "ratio")
- ... )
- >>> df.select(map_zip_with(
- ... "base", "ratio", lambda k, v1, v2: round(v1 * v2, 2)).alias("updated_data")
- ... ).show(truncate=False)
- +---------------------------+
- |updated_data |
- +---------------------------+
- |{SALES -> 16.8, IT -> 48.0}|
- +---------------------------+
- """
- return _invoke_higher_order_function("map_zip_with", [col1, col2], [f])
+map_zip_with.__doc__ = pysparkfuncs.map_zip_with.__doc__
def posexplode(col: "ColumnOrName") -> Column:
- """
- Returns a new row for each element with position in the given array or map.
- Uses the default column name `pos` for position, and `col` for elements in the
- array and `key` and `value` for elements in the map unless specified otherwise.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("posexplode", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- one row per array item or map key value including positions as a separate column.
-
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> eDF = spark.createDataFrame([Row(a=1, intlist=[1,2,3], mapfield={"a": "b"})])
- >>> eDF.select(posexplode(eDF.intlist)).collect()
- [Row(pos=0, col=1), Row(pos=1, col=2), Row(pos=2, col=3)]
-
- >>> eDF.select(posexplode(eDF.mapfield)).show()
- +---+---+-----+
- |pos|key|value|
- +---+---+-----+
- | 0| a| b|
- +---+---+-----+
- """
- return _invoke_function_over_columns("posexplode", col)
+posexplode.__doc__ = pysparkfuncs.posexplode.__doc__
def posexplode_outer(col: "ColumnOrName") -> Column:
- """
- Returns a new row for each element with position in the given array or map.
- Unlike posexplode, if the array/map is null or empty then the row (null, null) is produced.
- Uses the default column name `pos` for position, and `col` for elements in the
- array and `key` and `value` for elements in the map unless specified otherwise.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("posexplode_outer", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- one row per array item or map key value including positions as a separate column.
-
- Examples
- --------
- >>> df = spark.createDataFrame(
- ... [(1, ["foo", "bar"], {"x": 1.0}), (2, [], {}), (3, None, None)],
- ... ("id", "an_array", "a_map")
- ... )
- >>> df.select("id", "an_array", posexplode_outer("a_map")).show()
- +---+----------+----+----+-----+
- | id| an_array| pos| key|value|
- +---+----------+----+----+-----+
- | 1|[foo, bar]| 0| x| 1.0|
- | 2| []|null|null| null|
- | 3| null|null|null| null|
- +---+----------+----+----+-----+
- >>> df.select("id", "a_map", posexplode_outer("an_array")).show()
- +---+----------+----+----+
- | id| a_map| pos| col|
- +---+----------+----+----+
- | 1|{x -> 1.0}| 0| foo|
- | 1|{x -> 1.0}| 1| bar|
- | 2| {}|null|null|
- | 3| null|null|null|
- +---+----------+----+----+
- """
- return _invoke_function_over_columns("posexplode_outer", col)
+posexplode_outer.__doc__ = pysparkfuncs.posexplode_outer.__doc__
def reverse(col: "ColumnOrName") -> Column:
- """
- Collection function: returns a reversed string or an array with reverse order of elements.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("reverse", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- Returns
- -------
- :class:`~pyspark.sql.Column`
- array of elements in reverse order.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('Spark SQL',)], ['data'])
- >>> df.select(reverse(df.data).alias('s')).collect()
- [Row(s='LQS krapS')]
- >>> df = spark.createDataFrame([([2, 1, 3],) ,([1],) ,([],)], ['data'])
- >>> df.select(reverse(df.data).alias('r')).collect()
- [Row(r=[3, 1, 2]), Row(r=[1]), Row(r=[])]
- """
- return _invoke_function_over_columns("reverse", col)
+reverse.__doc__ = pysparkfuncs.reverse.__doc__
# TODO(SPARK-41493): Support options
def schema_of_csv(csv: "ColumnOrName") -> Column:
- """
- Parses a CSV string and infers its schema in DDL format.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- csv : :class:`~pyspark.sql.Column` or str
- a CSV string or a foldable string column containing a CSV string.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a string representation of a :class:`StructType` parsed from given CSV.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(schema_of_csv(lit('1|a'), {'sep':'|'}).alias("csv")).collect()
- [Row(csv='STRUCT<_c0: INT, _c1: STRING>')]
- >>> df.select(schema_of_csv('1|a', {'sep':'|'}).alias("csv")).collect()
- [Row(csv='STRUCT<_c0: INT, _c1: STRING>')]
- """
-
if isinstance(csv, Column):
_csv = csv
elif isinstance(csv, str):
@@ -5426,33 +1426,11 @@ def schema_of_csv(csv: "ColumnOrName") -> Column:
return _invoke_function("schema_of_csv", _csv)
-# TODO(SPARK-41494): Support options
-def schema_of_json(json: "ColumnOrName") -> Column:
- """
- Parses a JSON string and infers its schema in DDL format.
-
- .. versionadded:: 2.4.0
-
- Parameters
- ----------
- json : :class:`~pyspark.sql.Column` or str
- a JSON string or a foldable string column containing a JSON string.
+schema_of_csv.__doc__ = pysparkfuncs.schema_of_csv.__doc__
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a string representation of a :class:`StructType` parsed from given JSON.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(schema_of_json(lit('{"a": 0}')).alias("json")).collect()
- [Row(json='STRUCT<a: BIGINT>')]
- >>> schema = schema_of_json('{a: 1}', {'allowUnquotedFieldNames':'true'})
- >>> df.select(schema.alias("json")).collect()
- [Row(json='STRUCT<a: BIGINT>')]
- """
+# TODO(SPARK-41494): Support options
+def schema_of_json(json: "ColumnOrName") -> Column:
if isinstance(json, Column):
_json = json
elif isinstance(json, str):
@@ -5463,89 +1441,26 @@ def schema_of_json(json: "ColumnOrName") -> Column:
return _invoke_function("schema_of_json", _json)
-def shuffle(col: "ColumnOrName") -> Column:
- """
- Collection function: Generates a random permutation of the given array.
+schema_of_json.__doc__ = pysparkfuncs.schema_of_json.__doc__
- .. versionadded:: 3.4.0
- Notes
- -----
- The function is non-deterministic.
+def shuffle(col: "ColumnOrName") -> Column:
+ return _invoke_function_over_columns("shuffle", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- Returns
- -------
- :class:`~pyspark.sql.Column`
- an array of elements in random order.
-
- Examples
- --------
- >>> df = spark.createDataFrame([([1, 20, 3, 5],), ([1, 20, None, 3],)], ['data'])
- >>> df.select(shuffle(df.data).alias('s')).collect() # doctest: +SKIP
- [Row(s=[3, 1, 5, 20]), Row(s=[20, None, 3, 1])]
- """
- return _invoke_function_over_columns("shuffle", col)
+shuffle.__doc__ = pysparkfuncs.shuffle.__doc__
def size(col: "ColumnOrName") -> Column:
- """
- Collection function: returns the length of the array or map stored in the column.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("size", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- Returns
- -------
- :class:`~pyspark.sql.Column`
- length of the array/map.
-
- Examples
- --------
- >>> df = spark.createDataFrame([([1, 2, 3],),([1],),([],)], ['data'])
- >>> df.select(size(df.data)).collect()
- [Row(size(data)=3), Row(size(data)=1), Row(size(data)=0)]
- """
- return _invoke_function_over_columns("size", col)
+size.__doc__ = pysparkfuncs.size.__doc__
def slice(
col: "ColumnOrName", start: Union["ColumnOrName", int], length: Union["ColumnOrName", int]
) -> Column:
- """
- Collection function: returns an array containing all the elements in `x` from index `start`
- (array indices start at 1, or from the end if `start` is negative) with the specified `length`.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- column name or column containing the array to be sliced
- start : :class:`~pyspark.sql.Column` or str or int
- column name, column, or int containing the starting index
- length : :class:`~pyspark.sql.Column` or str or int
- column name, column, or int containing the length of the slice
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a column of array type. Subset of array.
-
- Examples
- --------
- >>> df = spark.createDataFrame([([1, 2, 3],), ([4, 5],)], ['x'])
- >>> df.select(slice(df.x, 2, 2).alias("sliced")).collect()
- [Row(sliced=[2, 3]), Row(sliced=[5])]
- """
if isinstance(start, Column):
_start = start
elif isinstance(start, int):
@@ -5563,276 +1478,64 @@ def slice(
return _invoke_function("slice", _to_col(col), _start, _length)
-def sort_array(col: "ColumnOrName", asc: bool = True) -> Column:
- """
- Collection function: sorts the input array in ascending or descending order according
- to the natural ordering of the array elements. Null elements will be placed at the beginning
- of the returned array in ascending order or at the end of the returned array in descending
- order.
-
- .. versionadded:: 3.4.0
+slice.__doc__ = pysparkfuncs.slice.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- asc : bool, optional
- whether to sort in ascending or descending order. If `asc` is True (default)
- then ascending and if False then descending.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- sorted array.
-
- Examples
- --------
- >>> df = spark.createDataFrame([([2, 1, None, 3],),([1],),([],)], ['data'])
- >>> df.select(sort_array(df.data).alias('r')).collect()
- [Row(r=[None, 1, 2, 3]), Row(r=[1]), Row(r=[])]
- >>> df.select(sort_array(df.data, asc=False).alias('r')).collect()
- [Row(r=[3, 2, 1, None]), Row(r=[1]), Row(r=[])]
- """
+def sort_array(col: "ColumnOrName", asc: bool = True) -> Column:
return _invoke_function("sort_array", _to_col(col), lit(asc))
+sort_array.__doc__ = pysparkfuncs.sort_array.__doc__
+
+
def struct(
*cols: Union["ColumnOrName", List["ColumnOrName"], Tuple["ColumnOrName", ...]]
) -> Column:
- """Creates a new struct column.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- cols : list, set, str or :class:`~pyspark.sql.Column`
- column names or :class:`~pyspark.sql.Column`\\s to contain in the output struct.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a struct type column of given columns.
-
- Examples
- --------
- >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5)], ("name", "age"))
- >>> df.select(struct('age', 'name').alias("struct")).collect()
- [Row(struct=Row(age=2, name='Alice')), Row(struct=Row(age=5, name='Bob'))]
- >>> df.select(struct([df.age, df.name]).alias("struct")).collect()
- [Row(struct=Row(age=2, name='Alice')), Row(struct=Row(age=5, name='Bob'))]
- """
if len(cols) == 1 and isinstance(cols[0], (list, set, tuple)):
cols = cols[0] # type: ignore[assignment]
return _invoke_function_over_columns("struct", *cols) # type: ignore[arg-type]
-# TODO(SPARK-41493): Support options
-def to_csv(col: "ColumnOrName") -> Column:
- """
- Converts a column containing a :class:`StructType` into a CSV string.
- Throws an exception, in the case of an unsupported type.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column containing a struct.
+struct.__doc__ = pysparkfuncs.struct.__doc__
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a CSV string converted from given :class:`StructType`.
-
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> data = [(1, Row(age=2, name='Alice'))]
- >>> df = spark.createDataFrame(data, ("key", "value"))
- >>> df.select(to_csv(df.value).alias("csv")).collect()
- [Row(csv='2,Alice')]
- """
+# TODO(SPARK-41493): Support options
+def to_csv(col: "ColumnOrName") -> Column:
return _invoke_function("to_csv", _to_col(col))
-# TODO(SPARK-41494): Support options
-def to_json(col: "ColumnOrName") -> Column:
- """
- Converts a column containing a :class:`StructType`, :class:`ArrayType` or a :class:`MapType`
- into a JSON string. Throws an exception, in the case of an unsupported type.
+to_csv.__doc__ = pysparkfuncs.to_csv.__doc__
- .. versionadded:: 2.1.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column containing a struct, an array or a map.
+def to_json(col: "ColumnOrName") -> Column:
+ return _invoke_function("to_json", _to_col(col))
- Returns
- -------
- :class:`~pyspark.sql.Column`
- JSON object as string column.
-
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> from pyspark.sql.types import *
- >>> data = [(1, Row(age=2, name='Alice'))]
- >>> df = spark.createDataFrame(data, ("key", "value"))
- >>> df.select(to_json(df.value).alias("json")).collect()
- [Row(json='{"age":2,"name":"Alice"}')]
- >>> data = [(1, [Row(age=2, name='Alice'), Row(age=3, name='Bob')])]
- >>> df = spark.createDataFrame(data, ("key", "value"))
- >>> df.select(to_json(df.value).alias("json")).collect()
- [Row(json='[{"age":2,"name":"Alice"},{"age":3,"name":"Bob"}]')]
- >>> data = [(1, {"name": "Alice"})]
- >>> df = spark.createDataFrame(data, ("key", "value"))
- >>> df.select(to_json(df.value).alias("json")).collect()
- [Row(json='{"name":"Alice"}')]
- >>> data = [(1, [{"name": "Alice"}, {"name": "Bob"}])]
- >>> df = spark.createDataFrame(data, ("key", "value"))
- >>> df.select(to_json(df.value).alias("json")).collect()
- [Row(json='[{"name":"Alice"},{"name":"Bob"}]')]
- >>> data = [(1, ["Alice", "Bob"])]
- >>> df = spark.createDataFrame(data, ("key", "value"))
- >>> df.select(to_json(df.value).alias("json")).collect()
- [Row(json='["Alice","Bob"]')]
- """
- return _invoke_function("to_json", _to_col(col))
+to_json.__doc__ = pysparkfuncs.to_json.__doc__
def transform(
col: "ColumnOrName",
f: Union[Callable[[Column], Column], Callable[[Column, Column], Column]],
) -> Column:
- """
- Returns an array of elements after applying a transformation to each element in
- the input array.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- f : function
- a function that is applied to each element of the input array.
- Can take one of the following forms:
-
- - Unary ``(x: Column) -> Column: ...``
- - Binary ``(x: Column, i: Column) -> Column...``, where the second argument is
- a 0-based index of the element.
+ return _invoke_higher_order_function("transform", [col], [f])
- and can use methods of :class:`~pyspark.sql.Column`, functions defined in
- :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
- Python ``UserDefinedFunctions`` are not supported
- (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a new array of transformed elements.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(1, [1, 2, 3, 4])], ("key", "values"))
- >>> df.select(transform("values", lambda x: x * 2).alias("doubled")).show()
- +------------+
- | doubled|
- +------------+
- |[2, 4, 6, 8]|
- +------------+
-
- >>> def alternate(x, i):
- ... return when(i % 2 == 0, x).otherwise(-x)
- >>> df.select(transform("values", alternate).alias("alternated")).show()
- +--------------+
- | alternated|
- +--------------+
- |[1, -2, 3, -4]|
- +--------------+
- """
- return _invoke_higher_order_function("transform", [col], [f])
+transform.__doc__ = pysparkfuncs.transform.__doc__
def transform_keys(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column:
- """
- Applies a function to every key-value pair in a map and returns
- a map with the results of those applications as the new keys for the pairs.
-
- .. versionadded:: 3.4.0
+ return _invoke_higher_order_function("transform_keys", [col], [f])
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- f : function
- a binary function ``(k: Column, v: Column) -> Column...``
- Can use methods of :class:`~pyspark.sql.Column`, functions defined in
- :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
- Python ``UserDefinedFunctions`` are not supported
- (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a new map of enties where new keys were calculated by applying given function to
- each key value argument.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(1, {"foo": -2.0, "bar": 2.0})], ("id", "data"))
- >>> df.select(transform_keys(
- ... "data", lambda k, _: upper(k)).alias("data_upper")
- ... ).show(truncate=False)
- +-------------------------+
- |data_upper |
- +-------------------------+
- |{BAR -> 2.0, FOO -> -2.0}|
- +-------------------------+
- """
- return _invoke_higher_order_function("transform_keys", [col], [f])
+transform_keys.__doc__ = pysparkfuncs.transform_keys.__doc__
def transform_values(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column:
- """
- Applies a function to every key-value pair in a map and returns
- a map with the results of those applications as the new values for the pairs.
-
- .. versionadded:: 3.4.0
+ return _invoke_higher_order_function("transform_values", [col], [f])
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- name of column or expression
- f : function
- a binary function ``(k: Column, v: Column) -> Column...``
- Can use methods of :class:`~pyspark.sql.Column`, functions defined in
- :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
- Python ``UserDefinedFunctions`` are not supported
- (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a new map of enties where new values were calculated by applying given function to
- each key value argument.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(1, {"IT": 10.0, "SALES": 2.0, "OPS": 24.0})], ("id", "data"))
- >>> df.select(transform_values(
- ... "data", lambda k, v: when(k.isin("IT", "OPS"), v + 10.0).otherwise(v)
- ... ).alias("new_data")).show(truncate=False)
- +---------------------------------------+
- |new_data |
- +---------------------------------------+
- |{OPS -> 34.0, IT -> 20.0, SALES -> 2.0}|
- +---------------------------------------+
- """
- return _invoke_higher_order_function("transform_values", [col], [f])
+transform_values.__doc__ = pysparkfuncs.transform_values.__doc__
def zip_with(
@@ -5840,481 +1543,111 @@ def zip_with(
right: "ColumnOrName",
f: Callable[[Column, Column], Column],
) -> Column:
- """
- Merge two given arrays, element-wise, into a single array using a function.
- If one array is shorter, nulls are appended at the end to match the length of the longer
- array, before applying the function.
-
- .. versionadded:: 3.4.0
+ return _invoke_higher_order_function("zip_with", [left, right], [f])
- Parameters
- ----------
- left : :class:`~pyspark.sql.Column` or str
- name of the first column or expression
- right : :class:`~pyspark.sql.Column` or str
- name of the second column or expression
- f : function
- a binary function ``(x1: Column, x2: Column) -> Column...``
- Can use methods of :class:`~pyspark.sql.Column`, functions defined in
- :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``.
- Python ``UserDefinedFunctions`` are not supported
- (`SPARK-27052 <https://issues.apache.org/jira/browse/SPARK-27052>`__).
- Returns
- -------
- :class:`~pyspark.sql.Column`
- array of calculated values derived by applying given function to each pair of arguments.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(1, [1, 3, 5, 8], [0, 2, 4, 6])], ("id", "xs", "ys"))
- >>> df.select(zip_with("xs", "ys", lambda x, y: x ** y).alias("powers")).show(truncate=False)
- +---------------------------+
- |powers |
- +---------------------------+
- |[1.0, 9.0, 625.0, 262144.0]|
- +---------------------------+
-
- >>> df = spark.createDataFrame([(1, ["foo", "bar"], [1, 2, 3])], ("id", "xs", "ys"))
- >>> df.select(zip_with("xs", "ys", lambda x, y: concat_ws("_", x, y)).alias("xs_ys")).show()
- +-----------------+
- | xs_ys|
- +-----------------+
- |[foo_1, bar_2, 3]|
- +-----------------+
- """
- return _invoke_higher_order_function("zip_with", [left, right], [f])
+zip_with.__doc__ = pysparkfuncs.zip_with.__doc__
# String/Binary functions
def upper(col: "ColumnOrName") -> Column:
- """
- Converts a string expression to upper case.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("upper", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- upper case values.
-
- Examples
- --------
- >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING")
- >>> df.select(upper("value")).show()
- +------------+
- |upper(value)|
- +------------+
- | SPARK|
- | PYSPARK|
- | PANDAS API|
- +------------+
- """
- return _invoke_function_over_columns("upper", col)
+upper.__doc__ = pysparkfuncs.upper.__doc__
def lower(col: "ColumnOrName") -> Column:
- """
- Converts a string expression to lower case.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("lower", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- lower case values.
-
- Examples
- --------
- >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING")
- >>> df.select(lower("value")).show()
- +------------+
- |lower(value)|
- +------------+
- | spark|
- | pyspark|
- | pandas api|
- +------------+
- """
- return _invoke_function_over_columns("lower", col)
+lower.__doc__ = pysparkfuncs.lower.__doc__
def ascii(col: "ColumnOrName") -> Column:
- """
- Computes the numeric value of the first character of the string column.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("ascii", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- numeric value.
-
- Examples
- --------
- >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING")
- >>> df.select(ascii("value")).show()
- +------------+
- |ascii(value)|
- +------------+
- | 83|
- | 80|
- | 80|
- +------------+
- """
- return _invoke_function_over_columns("ascii", col)
+ascii.__doc__ = pysparkfuncs.ascii.__doc__
def base64(col: "ColumnOrName") -> Column:
- """
- Computes the BASE64 encoding of a binary column and returns it as a string column.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("base64", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- BASE64 encoding of string value.
-
- Examples
- --------
- >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING")
- >>> df.select(base64("value")).show()
- +----------------+
- | base64(value)|
- +----------------+
- | U3Bhcms=|
- | UHlTcGFyaw==|
- |UGFuZGFzIEFQSQ==|
- +----------------+
- """
- return _invoke_function_over_columns("base64", col)
+base64.__doc__ = pysparkfuncs.base64.__doc__
def unbase64(col: "ColumnOrName") -> Column:
- """
- Decodes a BASE64 encoded string column and returns it as a binary column.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("unbase64", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- encoded string value.
-
- Examples
- --------
- >>> df = spark.createDataFrame(["U3Bhcms=",
- ... "UHlTcGFyaw==",
- ... "UGFuZGFzIEFQSQ=="], "STRING")
- >>> df.select(unbase64("value")).show()
- +--------------------+
- | unbase64(value)|
- +--------------------+
- | [53 70 61 72 6B]|
- |[50 79 53 70 61 7...|
- |[50 61 6E 64 61 7...|
- +--------------------+
- """
- return _invoke_function_over_columns("unbase64", col)
+unbase64.__doc__ = pysparkfuncs.unbase64.__doc__
def ltrim(col: "ColumnOrName") -> Column:
- """
- Trim the spaces from left end for the specified string value.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("ltrim", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- left trimmed values.
-
- Examples
- --------
- >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING")
- >>> df.select(ltrim("value").alias("r")).withColumn("length", length("r")).show()
- +-------+------+
- | r|length|
- +-------+------+
- | Spark| 5|
- |Spark | 7|
- | Spark| 5|
- +-------+------+
- """
- return _invoke_function_over_columns("ltrim", col)
+ltrim.__doc__ = pysparkfuncs.ltrim.__doc__
def rtrim(col: "ColumnOrName") -> Column:
- """
- Trim the spaces from right end for the specified string value.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("rtrim", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- right trimmed values.
-
- Examples
- --------
- >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING")
- >>> df.select(rtrim("value").alias("r")).withColumn("length", length("r")).show()
- +--------+------+
- | r|length|
- +--------+------+
- | Spark| 8|
- | Spark| 5|
- | Spark| 6|
- +--------+------+
- """
- return _invoke_function_over_columns("rtrim", col)
+rtrim.__doc__ = pysparkfuncs.rtrim.__doc__
def trim(col: "ColumnOrName") -> Column:
- """
- Trim the spaces from both ends for the specified string column.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("trim", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- trimmed values from both sides.
-
- Examples
- --------
- >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING")
- >>> df.select(trim("value").alias("r")).withColumn("length", length("r")).show()
- +-----+------+
- | r|length|
- +-----+------+
- |Spark| 5|
- |Spark| 5|
- |Spark| 5|
- +-----+------+
- """
- return _invoke_function_over_columns("trim", col)
+trim.__doc__ = pysparkfuncs.trim.__doc__
def concat_ws(sep: str, *cols: "ColumnOrName") -> Column:
- """
- Concatenates multiple input string columns together into a single string column,
- using the given separator.
-
- .. versionadded:: 3.4.0
+ return _invoke_function("concat_ws", lit(sep), *[_to_col(c) for c in cols])
- Parameters
- ----------
- sep : str
- words separator.
- cols : :class:`~pyspark.sql.Column` or str
- list of columns to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- string of concatenated words.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('abcd','123')], ['s', 'd'])
- >>> df.select(concat_ws('-', df.s, df.d).alias('s')).collect()
- [Row(s='abcd-123')]
- """
- return _invoke_function("concat_ws", lit(sep), *[_to_col(c) for c in cols])
+concat_ws.__doc__ = pysparkfuncs.concat_ws.__doc__
def decode(col: "ColumnOrName", charset: str) -> Column:
- """
- Computes the first argument into a string from a binary using the provided character set
- (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16').
-
- .. versionadded:: 3.4.0
+ return _invoke_function("decode", _to_col(col), lit(charset))
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- charset : str
- charset to use to decode to.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('abcd',)], ['a'])
- >>> df.select(decode("a", "UTF-8")).show()
- +----------------+
- |decode(a, UTF-8)|
- +----------------+
- | abcd|
- +----------------+
- """
- return _invoke_function("decode", _to_col(col), lit(charset))
+decode.__doc__ = pysparkfuncs.decode.__doc__
def encode(col: "ColumnOrName", charset: str) -> Column:
- """
- Computes the first argument into a binary from a string using the provided character set
- (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16').
-
- .. versionadded:: 3.4.0
+ return _invoke_function("encode", _to_col(col), lit(charset))
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- charset : str
- charset to use to encode.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('abcd',)], ['c'])
- >>> df.select(encode("c", "UTF-8")).show()
- +----------------+
- |encode(c, UTF-8)|
- +----------------+
- | [61 62 63 64]|
- +----------------+
- """
- return _invoke_function("encode", _to_col(col), lit(charset))
+encode.__doc__ = pysparkfuncs.encode.__doc__
# TODO(SPARK-41473): Resolve the data type mismatch issue and enable the function
# def format_number(col: "ColumnOrName", d: int) -> Column:
-# """
-# Formats the number X to a format like '#,--#,--#.--', rounded to d decimal places
-# with HALF_EVEN round mode, and returns the result as a string.
-#
-# .. versionadded:: 3.4.0
-#
-# Parameters
-# ----------
-# col : :class:`~pyspark.sql.Column` or str
-# the column name of the numeric value to be formatted
-# d : int
-# the N decimal places
-#
-# Returns
-# -------
-# :class:`~pyspark.sql.Column`
-# the column of formatted results.
-#
-# >>> spark.createDataFrame([(5,)], ['a']).select(format_number('a', 4).alias('v')).collect()
-# [Row(v='5.0000')]
-# """
# return _invoke_function("format_number", _to_col(col), lit(d))
+#
+# format_number.__doc__ = pysparkfuncs.format_number.__doc__
def format_string(format: str, *cols: "ColumnOrName") -> Column:
- """
- Formats the arguments in printf-style and returns the result as a string column.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- format : str
- string that can contain embedded format tags and used as result column's value
- cols : :class:`~pyspark.sql.Column` or str
- column names or :class:`~pyspark.sql.Column`\\s to be used in formatting
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column of formatted results.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(5, "hello")], ['a', 'b'])
- >>> df.select(format_string('%d %s', df.a, df.b).alias('v')).collect()
- [Row(v='5 hello')]
- """
return _invoke_function("format_string", lit(format), *[_to_col(c) for c in cols])
-def instr(str: "ColumnOrName", substr: str) -> Column:
- """
- Locate the position of the first occurrence of substr column in the given string.
- Returns null if either of the arguments are null.
+format_string.__doc__ = pysparkfuncs.format_string.__doc__
- .. versionadded:: 3.4.0
- Notes
- -----
- The position is not zero based, but 1 based index. Returns 0 if substr
- could not be found in str.
+def instr(str: "ColumnOrName", substr: str) -> Column:
+ return _invoke_function("instr", _to_col(str), lit(substr))
- Parameters
- ----------
- str : :class:`~pyspark.sql.Column` or str
- target column to work on.
- substr : str
- substring to look for.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- location of the first occurence of the substring as integer.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('abcd',)], ['s',])
- >>> df.select(instr(df.s, 'b').alias('s')).collect()
- [Row(s=2)]
- """
- return _invoke_function("instr", _to_col(str), lit(substr))
+instr.__doc__ = pysparkfuncs.instr.__doc__
def overlay(
@@ -6323,39 +1656,6 @@ def overlay(
pos: Union["ColumnOrName", int],
len: Union["ColumnOrName", int] = -1,
) -> Column:
- """
- Overlay the specified portion of `src` with `replace`,
- starting from byte position `pos` of `src` and proceeding for `len` bytes.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- src : :class:`~pyspark.sql.Column` or str
- column name or column containing the string that will be replaced
- replace : :class:`~pyspark.sql.Column` or str
- column name or column containing the substitution string
- pos : :class:`~pyspark.sql.Column` or str or int
- column name, column, or int containing the starting position in src
- len : :class:`~pyspark.sql.Column` or str or int, optional
- column name, column, or int containing the number of bytes to replace in src
- string by 'replace' defaults to -1, which represents the length of the 'replace' string
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- string with replaced values.
-
- Examples
- --------
- >>> df = spark.createDataFrame([("SPARK_SQL", "CORE")], ("x", "y"))
- >>> df.select(overlay("x", "y", 7).alias("overlayed")).collect()
- [Row(overlayed='SPARK_CORE')]
- >>> df.select(overlay("x", "y", 7, 0).alias("overlayed")).collect()
- [Row(overlayed='SPARK_CORESQL')]
- >>> df.select(overlay("x", "y", 7, 2).alias("overlayed")).collect()
- [Row(overlayed='SPARK_COREL')]
- """
if not isinstance(pos, (int, str, Column)):
raise TypeError(
"pos should be an integer or a Column / column name, got {}".format(type(pos))
@@ -6373,372 +1673,89 @@ def overlay(
return _invoke_function_over_columns("overlay", src, replace, pos, len)
-def sentences(
- string: "ColumnOrName",
- language: Optional["ColumnOrName"] = None,
- country: Optional["ColumnOrName"] = None,
-) -> Column:
- """
- Splits a string into arrays of sentences, where each sentence is an array of words.
- The 'language' and 'country' arguments are optional, and if omitted, the default locale is used.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- string : :class:`~pyspark.sql.Column` or str
- a string to be split
- language : :class:`~pyspark.sql.Column` or str, optional
- a language of the locale
- country : :class:`~pyspark.sql.Column` or str, optional
- a country of the locale
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- arrays of split sentences.
-
- Examples
- --------
- >>> df = spark.createDataFrame([["This is an example sentence."]], ["string"])
- >>> df.select(sentences(df.string, lit("en"), lit("US"))).show(truncate=False)
- +-----------------------------------+
- |sentences(string, en, US) |
- +-----------------------------------+
- |[[This, is, an, example, sentence]]|
- +-----------------------------------+
- >>> df = spark.createDataFrame([["Hello world. How are you?"]], ["s"])
- >>> df.select(sentences("s")).show(truncate=False)
- +---------------------------------+
- |sentences(s, , ) |
- +---------------------------------+
- |[[Hello, world], [How, are, you]]|
- +---------------------------------+
- """
+overlay.__doc__ = pysparkfuncs.overlay.__doc__
+
+
+def sentences(
+ string: "ColumnOrName",
+ language: Optional["ColumnOrName"] = None,
+ country: Optional["ColumnOrName"] = None,
+) -> Column:
_language = lit("") if language is None else _to_col(language)
_country = lit("") if country is None else _to_col(country)
return _invoke_function("sentences", _to_col(string), _language, _country)
-def substring(str: "ColumnOrName", pos: int, len: int) -> Column:
- """
- Substring starts at `pos` and is of length `len` when str is String type or
- returns the slice of byte array that starts at `pos` in byte and is of length `len`
- when str is Binary type.
+sentences.__doc__ = pysparkfuncs.sentences.__doc__
- .. versionadded:: 3.4.0
- Notes
- -----
- The position is not zero based, but 1 based index.
+def substring(str: "ColumnOrName", pos: int, len: int) -> Column:
+ return _invoke_function("substring", _to_col(str), lit(pos), lit(len))
- Parameters
- ----------
- str : :class:`~pyspark.sql.Column` or str
- target column to work on.
- pos : int
- starting position in str.
- len : int
- length of chars.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- substring of given value.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('abcd',)], ['s',])
- >>> df.select(substring(df.s, 1, 2).alias('s')).collect()
- [Row(s='ab')]
- """
- return _invoke_function("substring", _to_col(str), lit(pos), lit(len))
+substring.__doc__ = pysparkfuncs.substring.__doc__
def substring_index(str: "ColumnOrName", delim: str, count: int) -> Column:
- """
- Returns the substring from string str before count occurrences of the delimiter delim.
- If count is positive, everything the left of the final delimiter (counting from left) is
- returned. If count is negative, every to the right of the final delimiter (counting from the
- right) is returned. substring_index performs a case-sensitive match when searching for delim.
-
- .. versionadded:: 3.4.0
+ return _invoke_function("substring_index", _to_col(str), lit(delim), lit(count))
- Parameters
- ----------
- str : :class:`~pyspark.sql.Column` or str
- target column to work on.
- delim : str
- delimiter of values.
- count : int
- number of occurences.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- substring of given value.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('a.b.c.d',)], ['s'])
- >>> df.select(substring_index(df.s, '.', 2).alias('s')).collect()
- [Row(s='a.b')]
- >>> df.select(substring_index(df.s, '.', -3).alias('s')).collect()
- [Row(s='b.c.d')]
- """
- return _invoke_function("substring_index", _to_col(str), lit(delim), lit(count))
+substring_index.__doc__ = pysparkfuncs.substring_index.__doc__
def levenshtein(left: "ColumnOrName", right: "ColumnOrName") -> Column:
- """Computes the Levenshtein distance of the two given strings.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("levenshtein", left, right)
- Parameters
- ----------
- left : :class:`~pyspark.sql.Column` or str
- first column value.
- right : :class:`~pyspark.sql.Column` or str
- second column value.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- Levenshtein distance as integer value.
-
- Examples
- --------
- >>> df0 = spark.createDataFrame([('kitten', 'sitting',)], ['l', 'r'])
- >>> df0.select(levenshtein('l', 'r').alias('d')).collect()
- [Row(d=3)]
- """
- return _invoke_function_over_columns("levenshtein", left, right)
+levenshtein.__doc__ = pysparkfuncs.levenshtein.__doc__
def locate(substr: str, str: "ColumnOrName", pos: int = 1) -> Column:
- """
- Locate the position of the first occurrence of substr in a string column, after position pos.
-
- .. versionadded:: 3.4.0
+ return _invoke_function("locate", lit(substr), _to_col(str), lit(pos))
- Parameters
- ----------
- substr : str
- a string
- str : :class:`~pyspark.sql.Column` or str
- a Column of :class:`pyspark.sql.types.StringType`
- pos : int, optional
- start position (zero based)
- Returns
- -------
- :class:`~pyspark.sql.Column`
- position of the substring.
-
- Notes
- -----
- The position is not zero based, but 1 based index. Returns 0 if substr
- could not be found in str.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('abcd',)], ['s',])
- >>> df.select(locate('b', df.s, 1).alias('s')).collect()
- [Row(s=2)]
- """
- return _invoke_function("locate", lit(substr), _to_col(str), lit(pos))
+locate.__doc__ = pysparkfuncs.locate.__doc__
def lpad(col: "ColumnOrName", len: int, pad: str) -> Column:
- """
- Left-pad the string column to width `len` with `pad`.
-
- .. versionadded:: 3.4.0
+ return _invoke_function("lpad", _to_col(col), lit(len), lit(pad))
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- len : int
- length of the final string.
- pad : str
- chars to prepend.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- left padded result.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('abcd',)], ['s',])
- >>> df.select(lpad(df.s, 6, '#').alias('s')).collect()
- [Row(s='##abcd')]
- """
- return _invoke_function("lpad", _to_col(col), lit(len), lit(pad))
+lpad.__doc__ = pysparkfuncs.lpad.__doc__
def rpad(col: "ColumnOrName", len: int, pad: str) -> Column:
- """
- Right-pad the string column to width `len` with `pad`.
-
- .. versionadded:: 3.4.0
+ return _invoke_function("rpad", _to_col(col), lit(len), lit(pad))
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- len : int
- length of the final string.
- pad : str
- chars to append.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- right padded result.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('abcd',)], ['s',])
- >>> df.select(rpad(df.s, 6, '#').alias('s')).collect()
- [Row(s='abcd##')]
- """
- return _invoke_function("rpad", _to_col(col), lit(len), lit(pad))
+rpad.__doc__ = pysparkfuncs.rpad.__doc__
def repeat(col: "ColumnOrName", n: int) -> Column:
- """
- Repeats a string column n times, and returns it as a new string column.
-
- .. versionadded:: 3.4.0
+ return _invoke_function("repeat", _to_col(col), lit(n))
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- n : int
- number of times to repeat value.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- string with repeated values.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('ab',)], ['s',])
- >>> df.select(repeat(df.s, 3).alias('s')).collect()
- [Row(s='ababab')]
- """
- return _invoke_function("repeat", _to_col(col), lit(n))
+repeat.__doc__ = pysparkfuncs.repeat.__doc__
def split(str: "ColumnOrName", pattern: str, limit: int = -1) -> Column:
- """
- Splits str around matches of the given pattern.
-
- .. versionadded:: 3.4.0
+ return _invoke_function("split", _to_col(str), lit(pattern), lit(limit))
- Parameters
- ----------
- str : :class:`~pyspark.sql.Column` or str
- a string expression to split
- pattern : str
- a string representing a regular expression. The regex string should be
- a Java regular expression.
- limit : int, optional
- an integer which controls the number of times `pattern` is applied.
-
- * ``limit > 0``: The resulting array's length will not be more than `limit`, and the
- resulting array's last entry will contain all input beyond the last
- matched pattern.
- * ``limit <= 0``: `pattern` will be applied as many times as possible, and the resulting
- array can be of any size.
-
- `split` now takes an optional `limit` field. If not provided, default limit value is -1.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- array of separated strings.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('oneAtwoBthreeC',)], ['s',])
- >>> df.select(split(df.s, '[ABC]', 2).alias('s')).collect()
- [Row(s=['one', 'twoBthreeC'])]
- >>> df.select(split(df.s, '[ABC]', -1).alias('s')).collect()
- [Row(s=['one', 'two', 'three', ''])]
- """
- return _invoke_function("split", _to_col(str), lit(pattern), lit(limit))
+split.__doc__ = pysparkfuncs.split.__doc__
def regexp_extract(str: "ColumnOrName", pattern: str, idx: int) -> Column:
- r"""Extract a specific group matched by a Java regex, from the specified string column.
- If the regex did not match, or the specified group did not match, an empty string is returned.
-
- .. versionadded:: 3.4.0
+ return _invoke_function("regexp_extract", _to_col(str), lit(pattern), lit(idx))
- Parameters
- ----------
- str : :class:`~pyspark.sql.Column` or str
- target column to work on.
- pattern : str
- regex pattern to apply.
- idx : int
- matched group id.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- matched value specified by `idx` group id.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('100-200',)], ['str'])
- >>> df.select(regexp_extract('str', r'(\d+)-(\d+)', 1).alias('d')).collect()
- [Row(d='100')]
- >>> df = spark.createDataFrame([('foo',)], ['str'])
- >>> df.select(regexp_extract('str', r'(\d+)', 1).alias('d')).collect()
- [Row(d='')]
- >>> df = spark.createDataFrame([('aaaac',)], ['str'])
- >>> df.select(regexp_extract('str', '(a+)(b)?(c)', 2).alias('d')).collect()
- [Row(d='')]
- """
- return _invoke_function("regexp_extract", _to_col(str), lit(pattern), lit(idx))
+regexp_extract.__doc__ = pysparkfuncs.regexp_extract.__doc__
def regexp_replace(
string: "ColumnOrName", pattern: Union[str, Column], replacement: Union[str, Column]
) -> Column:
- r"""Replace all substrings of the specified string value that match regexp with replacement.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- string : :class:`~pyspark.sql.Column` or str
- column name or column containing the string value
- pattern : :class:`~pyspark.sql.Column` or str
- column object or str containing the regexp pattern
- replacement : :class:`~pyspark.sql.Column` or str
- column object or str containing the replacement
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- string with all substrings replaced.
-
- Examples
- --------
- >>> df = spark.createDataFrame([("100-200", r"(\d+)", "--")], ["str", "pattern", "replacement"])
- >>> df.select(regexp_replace('str', r'(\d+)', '--').alias('d')).collect()
- [Row(d='-----')]
- >>> df.select(regexp_replace("str", col("pattern"), col("replacement")).alias('d')).collect()
- [Row(d='-----')]
- """
if isinstance(pattern, str):
pattern = lit(pattern)
@@ -6748,161 +1765,49 @@ def regexp_replace(
return _invoke_function("regexp_replace", _to_col(string), pattern, replacement)
-def initcap(col: "ColumnOrName") -> Column:
- """Translate the first letter of each word to upper case in the sentence.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
+regexp_replace.__doc__ = pysparkfuncs.regexp_replace.__doc__
- Returns
- -------
- :class:`~pyspark.sql.Column`
- string with all first letters are uppercase in each word.
- Examples
- --------
- >>> spark.createDataFrame([('ab cd',)], ['a']).select(initcap("a").alias('v')).collect()
- [Row(v='Ab Cd')]
- """
+def initcap(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("initcap", col)
-def soundex(col: "ColumnOrName") -> Column:
- """
- Returns the SoundEx encoding for a string
+initcap.__doc__ = pysparkfuncs.initcap.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- SoundEx encoded string.
-
- Examples
- --------
- >>> df = spark.createDataFrame([("Peters",),("Uhrbach",)], ['name'])
- >>> df.select(soundex(df.name).alias("soundex")).collect()
- [Row(soundex='P362'), Row(soundex='U612')]
- """
+def soundex(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("soundex", col)
-def length(col: "ColumnOrName") -> Column:
- """Computes the character length of string data or number of bytes of binary data.
- The length of character data includes the trailing spaces. The length of binary data
- includes binary zeros.
+soundex.__doc__ = pysparkfuncs.soundex.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to work on.
+def length(col: "ColumnOrName") -> Column:
+ return _invoke_function_over_columns("length", col)
- Returns
- -------
- :class:`~pyspark.sql.Column`
- length of the value.
- Examples
- --------
- >>> spark.createDataFrame([('ABC ',)], ['a']).select(length('a').alias('length')).collect()
- [Row(length=4)]
- """
- return _invoke_function_over_columns("length", col)
+length.__doc__ = pysparkfuncs.length.__doc__
def octet_length(col: "ColumnOrName") -> Column:
- """
- Calculates the byte length for the specified string column.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("octet_length", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- Source column or strings
- Returns
- -------
- :class:`~pyspark.sql.Column`
- Byte length of the col
-
- Examples
- --------
- >>> from pyspark.sql.functions import octet_length
- >>> spark.createDataFrame([('cat',), ( '\U0001F408',)], ['cat']) \\
- ... .select(octet_length('cat')).collect()
- [Row(octet_length(cat)=3), Row(octet_length(cat)=4)]
- """
- return _invoke_function_over_columns("octet_length", col)
+octet_length.__doc__ = pysparkfuncs.octet_length.__doc__
def bit_length(col: "ColumnOrName") -> Column:
- """
- Calculates the bit length for the specified string column.
-
- .. versionadded:: 3.4.0
+ return _invoke_function_over_columns("bit_length", col)
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- Source column or strings
- Returns
- -------
- :class:`~pyspark.sql.Column`
- Bit length of the col
-
- Examples
- --------
- >>> from pyspark.sql.functions import bit_length
- >>> spark.createDataFrame([('cat',), ( '\U0001F408',)], ['cat']) \\
- ... .select(bit_length('cat')).collect()
- [Row(bit_length(cat)=24), Row(bit_length(cat)=32)]
- """
- return _invoke_function_over_columns("bit_length", col)
+bit_length.__doc__ = pysparkfuncs.bit_length.__doc__
def translate(srcCol: "ColumnOrName", matching: str, replace: str) -> Column:
- """A function translate any character in the `srcCol` by a character in `matching`.
- The characters in `replace` is corresponding to the characters in `matching`.
- Translation will happen whenever any character in the string is matching with the character
- in the `matching`.
-
- .. versionadded:: 3.4.0
+ return _invoke_function("translate", _to_col(srcCol), lit(matching), lit(replace))
- Parameters
- ----------
- srcCol : :class:`~pyspark.sql.Column` or str
- Source column or strings
- matching : str
- matching characters.
- replace : str
- characters for replacement. If this is shorter than `matching` string then
- those chars that don't have replacement will be dropped.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- replaced value.
-
- Examples
- --------
- >>> spark.createDataFrame([('translate',)], ['a']).select(translate('a', "rnlt", "123") \\
- ... .alias('r')).collect()
- [Row(r='1a2s3ae')]
- """
- return _invoke_function("translate", _to_col(srcCol), lit(matching), lit(replace))
+translate.__doc__ = pysparkfuncs.translate.__doc__
# Date/Timestamp functions
@@ -6912,605 +1817,158 @@ def translate(srcCol: "ColumnOrName", matching: str, replace: str) -> Column:
def current_date() -> Column:
- """
- Returns the current date at the start of query evaluation as a :class:`DateType` column.
- All calls of current_date within the same query return the same value.
-
- .. versionadded:: 3.4.0
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- current date.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(current_date()).show() # doctest: +SKIP
- +--------------+
- |current_date()|
- +--------------+
- | 2022-08-26|
- +--------------+
- """
return _invoke_function("current_date")
-def current_timestamp() -> Column:
- """
- Returns the current timestamp at the start of query evaluation as a :class:`TimestampType`
- column. All calls of current_timestamp within the same query return the same value.
+current_date.__doc__ = pysparkfuncs.current_date.__doc__
- .. versionadded:: 3.4.0
- Returns
- -------
- :class:`~pyspark.sql.Column`
- current date and time.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(current_timestamp()).show(truncate=False) # doctest: +SKIP
- +-----------------------+
- |current_timestamp() |
- +-----------------------+
- |2022-08-26 21:23:22.716|
- +-----------------------+
- """
+def current_timestamp() -> Column:
return _invoke_function("current_timestamp")
-def localtimestamp() -> Column:
- """
- Returns the current timestamp without time zone at the start of query evaluation
- as a timestamp without time zone column. All calls of localtimestamp within the
- same query return the same value.
+current_timestamp.__doc__ = pysparkfuncs.current_timestamp.__doc__
- .. versionadded:: 3.4.0
- Returns
- -------
- :class:`~pyspark.sql.Column`
- current local date and time.
-
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(localtimestamp()).show(truncate=False) # doctest: +SKIP
- +-----------------------+
- |localtimestamp() |
- +-----------------------+
- |2022-08-26 21:28:34.639|
- +-----------------------+
- """
+def localtimestamp() -> Column:
return _invoke_function("localtimestamp")
-def date_format(date: "ColumnOrName", format: str) -> Column:
- """
- Converts a date/timestamp/string to a value of string in the format specified by the date
- format given by the second argument.
-
- A pattern could be for instance `dd.MM.yyyy` and could return a string like '18.03.1993'. All
- pattern letters of `datetime pattern`_. can be used.
-
- .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
- .. versionadded:: 3.4.0
-
- Notes
- -----
- Whenever possible, use specialized functions like `year`.
+localtimestamp.__doc__ = pysparkfuncs.localtimestamp.__doc__
- Parameters
- ----------
- date : :class:`~pyspark.sql.Column` or str
- input column of values to format.
- format: str
- format to use to represent datetime values.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- string value representing formatted datetime.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
- >>> df.select(date_format('dt', 'MM/dd/yyy').alias('date')).collect()
- [Row(date='04/08/2015')]
- """
+def date_format(date: "ColumnOrName", format: str) -> Column:
return _invoke_function("date_format", _to_col(date), lit(format))
-def year(col: "ColumnOrName") -> Column:
- """
- Extract the year of a given date/timestamp as integer.
-
- .. versionadded:: 3.4.0
+date_format.__doc__ = pysparkfuncs.date_format.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target date/timestamp column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- year part of the date/timestamp as integer.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
- >>> df.select(year('dt').alias('year')).collect()
- [Row(year=2015)]
- """
+def year(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("year", col)
-def quarter(col: "ColumnOrName") -> Column:
- """
- Extract the quarter of a given date/timestamp as integer.
-
- .. versionadded:: 3.4.0
+year.__doc__ = pysparkfuncs.year.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target date/timestamp column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- quarter of the date/timestamp as integer.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
- >>> df.select(quarter('dt').alias('quarter')).collect()
- [Row(quarter=2)]
- """
+def quarter(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("quarter", col)
-def month(col: "ColumnOrName") -> Column:
- """
- Extract the month of a given date/timestamp as integer.
+quarter.__doc__ = pysparkfuncs.quarter.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target date/timestamp column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- month part of the date/timestamp as integer.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
- >>> df.select(month('dt').alias('month')).collect()
- [Row(month=4)]
- """
+def month(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("month", col)
-def dayofweek(col: "ColumnOrName") -> Column:
- """
- Extract the day of the week of a given date/timestamp as integer.
- Ranges from 1 for a Sunday through to 7 for a Saturday
-
- .. versionadded:: 3.4.0
+month.__doc__ = pysparkfuncs.month.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target date/timestamp column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- day of the week for given date/timestamp as integer.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
- >>> df.select(dayofweek('dt').alias('day')).collect()
- [Row(day=4)]
- """
+def dayofweek(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("dayofweek", col)
-def dayofmonth(col: "ColumnOrName") -> Column:
- """
- Extract the day of the month of a given date/timestamp as integer.
-
- .. versionadded:: 3.4.0
+dayofweek.__doc__ = pysparkfuncs.dayofweek.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target date/timestamp column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- day of the month for given date/timestamp as integer.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
- >>> df.select(dayofmonth('dt').alias('day')).collect()
- [Row(day=8)]
- """
+def dayofmonth(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("dayofmonth", col)
-def dayofyear(col: "ColumnOrName") -> Column:
- """
- Extract the day of the year of a given date/timestamp as integer.
-
- .. versionadded:: 3.4.0
+dayofmonth.__doc__ = pysparkfuncs.dayofmonth.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target date/timestamp column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- day of the year for given date/timestamp as integer.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
- >>> df.select(dayofyear('dt').alias('day')).collect()
- [Row(day=98)]
- """
+def dayofyear(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("dayofyear", col)
-def hour(col: "ColumnOrName") -> Column:
- """
- Extract the hours of a given timestamp as integer.
+dayofyear.__doc__ = pysparkfuncs.dayofyear.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target date/timestamp column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- hour part of the timestamp as integer.
-
- Examples
- --------
- >>> import datetime
- >>> df = spark.createDataFrame([(datetime.datetime(2015, 4, 8, 13, 8, 15),)], ['ts'])
- >>> df.select(hour('ts').alias('hour')).collect()
- [Row(hour=13)]
- """
+def hour(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("hour", col)
-def minute(col: "ColumnOrName") -> Column:
- """
- Extract the minutes of a given timestamp as integer.
-
- .. versionadded:: 3.4.0
+hour.__doc__ = pysparkfuncs.hour.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target date/timestamp column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- minutes part of the timestamp as integer.
-
- Examples
- --------
- >>> import datetime
- >>> df = spark.createDataFrame([(datetime.datetime(2015, 4, 8, 13, 8, 15),)], ['ts'])
- >>> df.select(minute('ts').alias('minute')).collect()
- [Row(minute=8)]
- """
+def minute(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("minute", col)
-def second(col: "ColumnOrName") -> Column:
- """
- Extract the seconds of a given date as integer.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target date/timestamp column to work on.
+minute.__doc__ = pysparkfuncs.minute.__doc__
- Returns
- -------
- :class:`~pyspark.sql.Column`
- `seconds` part of the timestamp as integer.
-
- Examples
- --------
- >>> import datetime
- >>> df = spark.createDataFrame([(datetime.datetime(2015, 4, 8, 13, 8, 15),)], ['ts'])
- >>> df.select(second('ts').alias('second')).collect()
- [Row(second=15)]
- """
- return _invoke_function_over_columns("second", col)
+def second(col: "ColumnOrName") -> Column:
+ return _invoke_function_over_columns("second", col)
-def weekofyear(col: "ColumnOrName") -> Column:
- """
- Extract the week number of a given date as integer.
- A week is considered to start on a Monday and week 1 is the first week with more than 3 days,
- as defined by ISO 8601
- .. versionadded:: 3.4.0
+second.__doc__ = pysparkfuncs.second.__doc__
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target timestamp column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- `week` of the year for given date as integer.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('2015-04-08',)], ['dt'])
- >>> df.select(weekofyear(df.dt).alias('week')).collect()
- [Row(week=15)]
- """
+def weekofyear(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("weekofyear", col)
-def make_date(year: "ColumnOrName", month: "ColumnOrName", day: "ColumnOrName") -> Column:
- """
- Returns a column with a date built from the year, month and day columns.
-
- .. versionadded:: 3.3.0
+weekofyear.__doc__ = pysparkfuncs.weekofyear.__doc__
- Parameters
- ----------
- year : :class:`~pyspark.sql.Column` or str
- The year to build the date
- month : :class:`~pyspark.sql.Column` or str
- The month to build the date
- day : :class:`~pyspark.sql.Column` or str
- The day to build the date
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a date built from given parts.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(2020, 6, 26)], ['Y', 'M', 'D'])
- >>> df.select(make_date(df.Y, df.M, df.D).alias("datefield")).collect()
- [Row(datefield=datetime.date(2020, 6, 26))]
- """
+def make_date(year: "ColumnOrName", month: "ColumnOrName", day: "ColumnOrName") -> Column:
return _invoke_function_over_columns("make_date", year, month, day)
-def date_add(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column:
- """
- Returns the date that is `days` days after `start`. If `days` is a negative value
- then these amount of days will be deducted from `start`.
-
- .. versionadded:: 3.4.0
+make_date.__doc__ = pysparkfuncs.make_date.__doc__
- Parameters
- ----------
- start : :class:`~pyspark.sql.Column` or str
- date column to work on.
- days : :class:`~pyspark.sql.Column` or str or int
- how many days after the given date to calculate.
- Accepts negative value as well to calculate backwards in time.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a date after/before given number of days.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('2015-04-08', 2,)], ['dt', 'add'])
- >>> df.select(date_add(df.dt, 1).alias('next_date')).collect()
- [Row(next_date=datetime.date(2015, 4, 9))]
- >>> df.select(date_add(df.dt, df.add.cast('integer')).alias('next_date')).collect()
- [Row(next_date=datetime.date(2015, 4, 10))]
- >>> df.select(date_add('dt', -1).alias('prev_date')).collect()
- [Row(prev_date=datetime.date(2015, 4, 7))]
- """
+def date_add(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column:
days = lit(days) if isinstance(days, int) else days
return _invoke_function_over_columns("date_add", start, days)
-def date_sub(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column:
- """
- Returns the date that is `days` days before `start`. If `days` is a negative value
- then these amount of days will be added to `start`.
+date_add.__doc__ = pysparkfuncs.date_add.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- start : :class:`~pyspark.sql.Column` or str
- date column to work on.
- days : :class:`~pyspark.sql.Column` or str or int
- how many days before the given date to calculate.
- Accepts negative value as well to calculate forward in time.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a date before/after given number of days.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('2015-04-08', 2,)], ['dt', 'sub'])
- >>> df.select(date_sub(df.dt, 1).alias('prev_date')).collect()
- [Row(prev_date=datetime.date(2015, 4, 7))]
- >>> df.select(date_sub(df.dt, df.sub.cast('integer')).alias('prev_date')).collect()
- [Row(prev_date=datetime.date(2015, 4, 6))]
- >>> df.select(date_sub('dt', -1).alias('next_date')).collect()
- [Row(next_date=datetime.date(2015, 4, 9))]
- """
+def date_sub(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column:
days = lit(days) if isinstance(days, int) else days
return _invoke_function_over_columns("date_sub", start, days)
-def datediff(end: "ColumnOrName", start: "ColumnOrName") -> Column:
- """
- Returns the number of days from `start` to `end`.
-
- .. versionadded:: 3.4.0
+date_sub.__doc__ = pysparkfuncs.date_sub.__doc__
- Parameters
- ----------
- end : :class:`~pyspark.sql.Column` or str
- to date column to work on.
- start : :class:`~pyspark.sql.Column` or str
- from date column to work on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- difference in days between two dates.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('2015-04-08','2015-05-10')], ['d1', 'd2'])
- >>> df.select(datediff(df.d2, df.d1).alias('diff')).collect()
- [Row(diff=32)]
- """
+def datediff(end: "ColumnOrName", start: "ColumnOrName") -> Column:
return _invoke_function_over_columns("datediff", end, start)
-def add_months(start: "ColumnOrName", months: Union["ColumnOrName", int]) -> Column:
- """
- Returns the date that is `months` months after `start`. If `months` is a negative value
- then these amount of months will be deducted from the `start`.
-
- .. versionadded:: 3.4.0
+datediff.__doc__ = pysparkfuncs.datediff.__doc__
- Parameters
- ----------
- start : :class:`~pyspark.sql.Column` or str
- date column to work on.
- months : :class:`~pyspark.sql.Column` or str or int
- how many months after the given date to calculate.
- Accepts negative value as well to calculate backwards.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- a date after/before given number of months.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('2015-04-08', 2)], ['dt', 'add'])
- >>> df.select(add_months(df.dt, 1).alias('next_month')).collect()
- [Row(next_month=datetime.date(2015, 5, 8))]
- >>> df.select(add_months(df.dt, df.add.cast('integer')).alias('next_month')).collect()
- [Row(next_month=datetime.date(2015, 6, 8))]
- >>> df.select(add_months('dt', -2).alias('prev_month')).collect()
- [Row(prev_month=datetime.date(2015, 2, 8))]
- """
+def add_months(start: "ColumnOrName", months: Union["ColumnOrName", int]) -> Column:
months = lit(months) if isinstance(months, int) else months
return _invoke_function_over_columns("add_months", start, months)
-def months_between(date1: "ColumnOrName", date2: "ColumnOrName", roundOff: bool = True) -> Column:
- """
- Returns number of months between dates date1 and date2.
- If date1 is later than date2, then the result is positive.
- A whole number is returned if both inputs have the same day of month or both are the last day
- of their respective months. Otherwise, the difference is calculated assuming 31 days per month.
- The result is rounded off to 8 digits unless `roundOff` is set to `False`.
-
- .. versionadded:: 3.4.0
+add_months.__doc__ = pysparkfuncs.add_months.__doc__
- Parameters
- ----------
- date1 : :class:`~pyspark.sql.Column` or str
- first date column.
- date2 : :class:`~pyspark.sql.Column` or str
- second date column.
- roundOff : bool, optional
- whether to round (to 8 digits) the final value or not (default: True).
- Returns
- -------
- :class:`~pyspark.sql.Column`
- number of months between two dates.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('1997-02-28 10:30:00', '1996-10-30')], ['date1', 'date2'])
- >>> df.select(months_between(df.date1, df.date2).alias('months')).collect()
- [Row(months=3.94959677)]
- >>> df.select(months_between(df.date1, df.date2, False).alias('months')).collect()
- [Row(months=3.9495967741935485)]
- """
+def months_between(date1: "ColumnOrName", date2: "ColumnOrName", roundOff: bool = True) -> Column:
return _invoke_function("months_between", _to_col(date1), _to_col(date2), lit(roundOff))
-def to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column:
- """Converts a :class:`~pyspark.sql.Column` into :class:`pyspark.sql.types.DateType`
- using the optionally specified format. Specify formats according to `datetime pattern`_.
- By default, it follows casting rules to :class:`pyspark.sql.types.DateType` if the format
- is omitted. Equivalent to ``col.cast("date")``.
-
- .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+months_between.__doc__ = pysparkfuncs.months_between.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- input column of values to convert.
- format: str, optional
- format to use to convert date values.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- date value as :class:`pyspark.sql.types.DateType` type.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['t'])
- >>> df.select(to_date(df.t).alias('date')).collect()
- [Row(date=datetime.date(1997, 2, 28))]
-
- >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['t'])
- >>> df.select(to_date(df.t, 'yyyy-MM-dd HH:mm:ss').alias('date')).collect()
- [Row(date=datetime.date(1997, 2, 28))]
- """
+def to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column:
if format is None:
return _invoke_function_over_columns("to_date", col)
else:
return _invoke_function("to_date", _to_col(col), lit(format))
+to_date.__doc__ = pysparkfuncs.to_date.__doc__
+
+
@overload
def to_timestamp(col: "ColumnOrName") -> Column:
...
@@ -7522,192 +1980,50 @@ def to_timestamp(col: "ColumnOrName", format: str) -> Column:
def to_timestamp(col: "ColumnOrName", format: Optional[str] = None) -> Column:
- """Converts a :class:`~pyspark.sql.Column` into :class:`pyspark.sql.types.TimestampType`
- using the optionally specified format. Specify formats according to `datetime pattern`_.
- By default, it follows casting rules to :class:`pyspark.sql.types.TimestampType` if the format
- is omitted. Equivalent to ``col.cast("timestamp")``.
-
- .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- column values to convert.
- format: str, optional
- format to use to convert timestamp values.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- timestamp value as :class:`pyspark.sql.types.TimestampType` type.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['t'])
- >>> df.select(to_timestamp(df.t).alias('dt')).collect()
- [Row(dt=datetime.datetime(1997, 2, 28, 10, 30))]
-
- >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['t'])
- >>> df.select(to_timestamp(df.t, 'yyyy-MM-dd HH:mm:ss').alias('dt')).collect()
- [Row(dt=datetime.datetime(1997, 2, 28, 10, 30))]
- """
if format is None:
return _invoke_function_over_columns("to_timestamp", col)
else:
return _invoke_function("to_timestamp", _to_col(col), lit(format))
-def trunc(date: "ColumnOrName", format: str) -> Column:
- """
- Returns date truncated to the unit specified by the format.
+to_timestamp.__doc__ = pysparkfuncs.to_timestamp.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- date : :class:`~pyspark.sql.Column` or str
- input column of values to truncate.
- format : str
- 'year', 'yyyy', 'yy' to truncate by year,
- or 'month', 'mon', 'mm' to truncate by month
- Other options are: 'week', 'quarter'
- Returns
- -------
- :class:`~pyspark.sql.Column`
- truncated date.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('1997-02-28',)], ['d'])
- >>> df.select(trunc(df.d, 'year').alias('year')).collect()
- [Row(year=datetime.date(1997, 1, 1))]
- >>> df.select(trunc(df.d, 'mon').alias('month')).collect()
- [Row(month=datetime.date(1997, 2, 1))]
- """
+def trunc(date: "ColumnOrName", format: str) -> Column:
return _invoke_function("trunc", _to_col(date), lit(format))
-def date_trunc(format: str, timestamp: "ColumnOrName") -> Column:
- """
- Returns timestamp truncated to the unit specified by the format.
-
- .. versionadded:: 3.4.0
+trunc.__doc__ = pysparkfuncs.trunc.__doc__
- Parameters
- ----------
- format : str
- 'year', 'yyyy', 'yy' to truncate by year,
- 'month', 'mon', 'mm' to truncate by month,
- 'day', 'dd' to truncate by day,
- Other options are:
- 'microsecond', 'millisecond', 'second', 'minute', 'hour', 'week', 'quarter'
- timestamp : :class:`~pyspark.sql.Column` or str
- input column of values to truncate.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- truncated timestamp.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('1997-02-28 05:02:11',)], ['t'])
- >>> df.select(date_trunc('year', df.t).alias('year')).collect()
- [Row(year=datetime.datetime(1997, 1, 1, 0, 0))]
- >>> df.select(date_trunc('mon', df.t).alias('month')).collect()
- [Row(month=datetime.datetime(1997, 2, 1, 0, 0))]
- """
+def date_trunc(format: str, timestamp: "ColumnOrName") -> Column:
return _invoke_function("date_trunc", lit(format), _to_col(timestamp))
-def next_day(date: "ColumnOrName", dayOfWeek: str) -> Column:
- """
- Returns the first date which is later than the value of the date column
- based on second `week day` argument.
-
- .. versionadded:: 3.4.0
+date_trunc.__doc__ = pysparkfuncs.date_trunc.__doc__
- Parameters
- ----------
- date : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- dayOfWeek : str
- day of the week, case-insensitive, accepts:
- "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column of computed results.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('2015-07-27',)], ['d'])
- >>> df.select(next_day(df.d, 'Sun').alias('date')).collect()
- [Row(date=datetime.date(2015, 8, 2))]
- """
+def next_day(date: "ColumnOrName", dayOfWeek: str) -> Column:
return _invoke_function("next_day", _to_col(date), lit(dayOfWeek))
-def last_day(date: "ColumnOrName") -> Column:
- """
- Returns the last day of the month which the given date belongs to.
-
- .. versionadded:: 3.4.0
+next_day.__doc__ = pysparkfuncs.next_day.__doc__
- Parameters
- ----------
- date : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- last day of the month.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('1997-02-10',)], ['d'])
- >>> df.select(last_day(df.d).alias('date')).collect()
- [Row(date=datetime.date(1997, 2, 28))]
- """
+def last_day(date: "ColumnOrName") -> Column:
return _invoke_function_over_columns("last_day", date)
-def from_unixtime(timestamp: "ColumnOrName", format: str = "yyyy-MM-dd HH:mm:ss") -> Column:
- """
- Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string
- representing the timestamp of that moment in the current system time zone in the given
- format.
+last_day.__doc__ = pysparkfuncs.last_day.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- timestamp : :class:`~pyspark.sql.Column` or str
- column of unix time values.
- format : str, optional
- format to use to convert to (default: yyyy-MM-dd HH:mm:ss)
- Returns
- -------
- :class:`~pyspark.sql.Column`
- formatted timestamp as string.
-
- Examples
- --------
- >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")
- >>> time_df = spark.createDataFrame([(1428476400,)], ['unix_time'])
- >>> time_df.select(from_unixtime('unix_time').alias('ts')).collect()
- [Row(ts='2015-04-08 00:00:00')]
- >>> spark.conf.unset("spark.sql.session.timeZone")
- """
+def from_unixtime(timestamp: "ColumnOrName", format: str = "yyyy-MM-dd HH:mm:ss") -> Column:
return _invoke_function("from_unixtime", _to_col(timestamp), lit(format))
+from_unixtime.__doc__ = pysparkfuncs.from_unixtime.__doc__
+
+
@overload
def unix_timestamp(timestamp: "ColumnOrName", format: str = ...) -> Column:
...
@@ -7721,432 +2037,101 @@ def unix_timestamp() -> Column:
def unix_timestamp(
timestamp: Optional["ColumnOrName"] = None, format: str = "yyyy-MM-dd HH:mm:ss"
) -> Column:
- """
- Convert time string with given pattern ('yyyy-MM-dd HH:mm:ss', by default)
- to Unix time stamp (in seconds), using the default timezone and the default
- locale, returns null if failed.
-
- if `timestamp` is None, then it returns current timestamp.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- timestamp : :class:`~pyspark.sql.Column` or str, optional
- timestamps of string values.
- format : str, optional
- alternative format to use for converting (default: yyyy-MM-dd HH:mm:ss).
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- unix time as long integer.
-
- Examples
- --------
- >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")
- >>> time_df = spark.createDataFrame([('2015-04-08',)], ['dt'])
- >>> time_df.select(unix_timestamp('dt', 'yyyy-MM-dd').alias('unix_time')).collect()
- [Row(unix_time=1428476400)]
- >>> spark.conf.unset("spark.sql.session.timeZone")
- """
if timestamp is None:
return _invoke_function("unix_timestamp")
return _invoke_function("unix_timestamp", _to_col(timestamp), lit(format))
-def from_utc_timestamp(timestamp: "ColumnOrName", tz: "ColumnOrName") -> Column:
- """
- This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function
- takes a timestamp which is timezone-agnostic, and interprets it as a timestamp in UTC, and
- renders that timestamp as a timestamp in the given time zone.
-
- However, timestamp in Spark represents number of microseconds from the Unix epoch, which is not
- timezone-agnostic. So in Spark this function just shift the timestamp value from UTC timezone to
- the given timezone.
-
- This function may return confusing result if the input is a string with timezone, e.g.
- '2018-03-13T06:18:23+00:00'. The reason is that, Spark firstly cast the string to timestamp
- according to the timezone in the string, and finally display the result by converting the
- timestamp to string according to the session local timezone.
-
- .. versionadded:: 3.4.0
+unix_timestamp.__doc__ = pysparkfuncs.unix_timestamp.__doc__
- Parameters
- ----------
- timestamp : :class:`~pyspark.sql.Column` or str
- the column that contains timestamps
- tz : :class:`~pyspark.sql.Column` or str
- A string detailing the time zone ID that the input should be adjusted to. It should
- be in the format of either region-based zone IDs or zone offsets. Region IDs must
- have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
- the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
- supported as aliases of '+00:00'. Other short names are not recommended to use
- because they can be ambiguous.
- `tz` can also take a :class:`~pyspark.sql.Column` containing timezone ID strings.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- timestamp value represented in given timezone.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('1997-02-28 10:30:00', 'JST')], ['ts', 'tz'])
- >>> df.select(from_utc_timestamp(df.ts, "PST").alias('local_time')).collect()
- [Row(local_time=datetime.datetime(1997, 2, 28, 2, 30))]
- >>> df.select(from_utc_timestamp(df.ts, df.tz).alias('local_time')).collect()
- [Row(local_time=datetime.datetime(1997, 2, 28, 19, 30))]
- """
+def from_utc_timestamp(timestamp: "ColumnOrName", tz: "ColumnOrName") -> Column:
if isinstance(tz, str):
tz = lit(tz)
return _invoke_function_over_columns("from_utc_timestamp", timestamp, tz)
-def to_utc_timestamp(timestamp: "ColumnOrName", tz: "ColumnOrName") -> Column:
- """
- This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function
- takes a timestamp which is timezone-agnostic, and interprets it as a timestamp in the given
- timezone, and renders that timestamp as a timestamp in UTC.
-
- However, timestamp in Spark represents number of microseconds from the Unix epoch, which is not
- timezone-agnostic. So in Spark this function just shift the timestamp value from the given
- timezone to UTC timezone.
-
- This function may return confusing result if the input is a string with timezone, e.g.
- '2018-03-13T06:18:23+00:00'. The reason is that, Spark firstly cast the string to timestamp
- according to the timezone in the string, and finally display the result by converting the
- timestamp to string according to the session local timezone.
+from_utc_timestamp.__doc__ = pysparkfuncs.from_utc_timestamp.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- timestamp : :class:`~pyspark.sql.Column` or str
- the column that contains timestamps
- tz : :class:`~pyspark.sql.Column` or str
- A string detailing the time zone ID that the input should be adjusted to. It should
- be in the format of either region-based zone IDs or zone offsets. Region IDs must
- have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
- the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
- supported as aliases of '+00:00'. Other short names are not recommended to use
- because they can be ambiguous.
- `tz` can also take a :class:`~pyspark.sql.Column` containing timezone ID strings.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- timestamp value represented in UTC timezone.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('1997-02-28 10:30:00', 'JST')], ['ts', 'tz'])
- >>> df.select(to_utc_timestamp(df.ts, "PST").alias('utc_time')).collect()
- [Row(utc_time=datetime.datetime(1997, 2, 28, 18, 30))]
- >>> df.select(to_utc_timestamp(df.ts, df.tz).alias('utc_time')).collect()
- [Row(utc_time=datetime.datetime(1997, 2, 28, 1, 30))]
- """
+def to_utc_timestamp(timestamp: "ColumnOrName", tz: "ColumnOrName") -> Column:
if isinstance(tz, str):
tz = lit(tz)
return _invoke_function_over_columns("to_utc_timestamp", timestamp, tz)
-def timestamp_seconds(col: "ColumnOrName") -> Column:
- """
- Converts the number of seconds from the Unix epoch (1970-01-01T00:00:00Z)
- to a timestamp.
+to_utc_timestamp.__doc__ = pysparkfuncs.to_utc_timestamp.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- unix time values.
+def timestamp_seconds(col: "ColumnOrName") -> Column:
+ return _invoke_function_over_columns("timestamp_seconds", col)
- Returns
- -------
- :class:`~pyspark.sql.Column`
- converted timestamp value.
-
- Examples
- --------
- >>> from pyspark.sql.functions import timestamp_seconds
- >>> spark.conf.set("spark.sql.session.timeZone", "UTC")
- >>> time_df = spark.createDataFrame([(1230219000,)], ['unix_time'])
- >>> time_df.select(timestamp_seconds(time_df.unix_time).alias('ts')).show()
- +-------------------+
- | ts|
- +-------------------+
- |2008-12-25 15:30:00|
- +-------------------+
- >>> time_df.select(timestamp_seconds('unix_time').alias('ts')).printSchema()
- root
- |-- ts: timestamp (nullable = true)
- >>> spark.conf.unset("spark.sql.session.timeZone")
- """
- return _invoke_function_over_columns("timestamp_seconds", col)
+timestamp_seconds.__doc__ = pysparkfuncs.timestamp_seconds.__doc__
# Misc Functions
def assert_true(col: "ColumnOrName", errMsg: Optional[Union[Column, str]] = None) -> Column:
- """
- Returns `null` if the input column is `true`; throws an exception
- with the provided error message otherwise.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- column name or column that represents the input column to test
- errMsg : :class:`~pyspark.sql.Column` or str, optional
- A Python string literal or column containing the error message
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- `null` if the input column is `true` otherwise throws an error with specified message.
-
- Examples
- --------
- >>> df = spark.createDataFrame([(0,1)], ['a', 'b'])
- >>> df.select(assert_true(df.a < df.b).alias('r')).collect()
- [Row(r=None)]
- >>> df.select(assert_true(df.a < df.b, df.a).alias('r')).collect()
- [Row(r=None)]
- >>> df.select(assert_true(df.a < df.b, 'error').alias('r')).collect()
- [Row(r=None)]
- >>> df.select(assert_true(df.a > df.b, 'My error msg').alias('r')).collect() # doctest: +SKIP
- ...
- java.lang.RuntimeException: My error msg
- ...
- """
if errMsg is None:
return _invoke_function_over_columns("assert_true", col)
if not isinstance(errMsg, (str, Column)):
raise TypeError("errMsg should be a Column or a str, got {}".format(type(errMsg)))
-
_err_msg = lit(errMsg) if isinstance(errMsg, str) else _to_col(errMsg)
-
return _invoke_function("assert_true", _to_col(col), _err_msg)
-def raise_error(errMsg: Union[Column, str]) -> Column:
- """
- Throws an exception with the provided error message.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- errMsg : :class:`~pyspark.sql.Column` or str
- A Python string literal or column containing the error message
+assert_true.__doc__ = pysparkfuncs.assert_true.__doc__
- Returns
- -------
- :class:`~pyspark.sql.Column`
- throws an error with specified message.
- Examples
- --------
- >>> df = spark.range(1)
- >>> df.select(raise_error("My error message")).show() # doctest: +SKIP
- ...
- java.lang.RuntimeException: My error message
- ...
- """
+def raise_error(errMsg: Union[Column, str]) -> Column:
if not isinstance(errMsg, (str, Column)):
raise TypeError("errMsg should be a Column or a str, got {}".format(type(errMsg)))
-
_err_msg = lit(errMsg) if isinstance(errMsg, str) else _to_col(errMsg)
-
return _invoke_function("raise_error", _err_msg)
-def crc32(col: "ColumnOrName") -> Column:
- """
- Calculates the cyclic redundancy check value (CRC32) of a binary column and
- returns the value as a bigint.
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
-
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
+raise_error.__doc__ = pysparkfuncs.raise_error.__doc__
- .. versionadded:: 3.4.0
- Examples
- --------
- >>> spark.createDataFrame([('ABC',)], ['a']).select(crc32('a').alias('crc32')).collect()
- [Row(crc32=2743272264)]
- """
+def crc32(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("crc32", col)
-def hash(*cols: "ColumnOrName") -> Column:
- """Calculates the hash code of given columns, and returns the result as an int column.
-
- .. versionadded:: 2.0.0
+crc32.__doc__ = pysparkfuncs.crc32.__doc__
- Parameters
- ----------
- cols : :class:`~pyspark.sql.Column` or str
- one or more columns to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- hash value as int column.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('ABC', 'DEF')], ['c1', 'c2'])
-
- Hash for one column
-
- >>> df.select(hash('c1').alias('hash')).show()
- +----------+
- | hash|
- +----------+
- |-757602832|
- +----------+
-
- Two or more columns
-
- >>> df.select(hash('c1', 'c2').alias('hash')).show()
- +---------+
- | hash|
- +---------+
- |599895104|
- +---------+
- """
+def hash(*cols: "ColumnOrName") -> Column:
return _invoke_function_over_columns("hash", *cols)
-def xxhash64(*cols: "ColumnOrName") -> Column:
- """Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm,
- and returns the result as a long column. The hash computation uses an initial seed of 42.
-
- .. versionadded:: 3.0.0
+hash.__doc__ = pysparkfuncs.hash.__doc__
- Parameters
- ----------
- cols : :class:`~pyspark.sql.Column` or str
- one or more columns to compute on.
- Returns
- -------
- :class:`~pyspark.sql.Column`
- hash value as long column.
-
- Examples
- --------
- >>> df = spark.createDataFrame([('ABC', 'DEF')], ['c1', 'c2'])
-
- Hash for one column
-
- >>> df.select(xxhash64('c1').alias('hash')).show()
- +-------------------+
- | hash|
- +-------------------+
- |4105715581806190027|
- +-------------------+
-
- Two or more columns
-
- >>> df.select(xxhash64('c1', 'c2').alias('hash')).show()
- +-------------------+
- | hash|
- +-------------------+
- |3233247871021311208|
- +-------------------+
- """
+def xxhash64(*cols: "ColumnOrName") -> Column:
return _invoke_function_over_columns("xxhash64", *cols)
-def md5(col: "ColumnOrName") -> Column:
- """Calculates the MD5 digest and returns the value as a 32 character hex string.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
+xxhash64.__doc__ = pysparkfuncs.xxhash64.__doc__
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
- Examples
- --------
- >>> spark.createDataFrame([('ABC',)], ['a']).select(md5('a').alias('hash')).collect()
- [Row(hash='902fbdd2b1df0c4f70b4a5d23525e932')]
- """
+def md5(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("md5", col)
-def sha1(col: "ColumnOrName") -> Column:
- """Returns the hex string result of SHA-1.
+md5.__doc__ = pysparkfuncs.md5.__doc__
- .. versionadded:: 3.4.0
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
+def sha1(col: "ColumnOrName") -> Column:
+ return _invoke_function_over_columns("sha1", col)
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
- Examples
- --------
- >>> spark.createDataFrame([('ABC',)], ['a']).select(sha1('a').alias('hash')).collect()
- [Row(hash='3c01bdbb26f358bab27f267924aa2c9a03fcfdb8')]
- """
- return _invoke_function_over_columns("sha1", col)
+sha1.__doc__ = pysparkfuncs.sha1.__doc__
def sha2(col: "ColumnOrName", numBits: int) -> Column:
- """Returns the hex string result of SHA-2 family of hash functions (SHA-224, SHA-256, SHA-384,
- and SHA-512). The numBits indicates the desired bit length of the result, which must have a
- value of 224, 256, 384, 512, or 0 (which is equivalent to 256).
-
- .. versionadded:: 3.4.0
+ return _invoke_function("sha2", _to_col(col), lit(numBits))
- Parameters
- ----------
- col : :class:`~pyspark.sql.Column` or str
- target column to compute on.
- numBits : int
- the desired bit length of the result, which must have a
- value of 224, 256, 384, 512, or 0 (which is equivalent to 256).
- Returns
- -------
- :class:`~pyspark.sql.Column`
- the column for computed results.
-
- Examples
- --------
- >>> df = spark.createDataFrame([["Alice"], ["Bob"]], ["name"])
- >>> df.withColumn("sha2", sha2(df.name, 256)).show(truncate=False)
- +-----+----------------------------------------------------------------+
- |name |sha2 |
- +-----+----------------------------------------------------------------+
- |Alice|3bc51062973c458d5a6f2d8d64a023246354ad7e064b1e4e009ec8a0699a3043|
- |Bob |cd9fb1e148ccd8442e5aa74904cc73bf6fb54d1d54d333bd596aa9bb4bb4e961|
- +-----+----------------------------------------------------------------+
- """
- return _invoke_function("sha2", _to_col(col), lit(numBits))
+sha2.__doc__ = pysparkfuncs.sha2.__doc__
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index f04b0a61438..9072985f209 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -132,6 +132,9 @@ def lit(col: Any) -> Column:
.. versionadded:: 1.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column`, str, int, float, bool or list.
@@ -186,6 +189,9 @@ def col(col: str) -> Column:
.. versionadded:: 1.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : str
@@ -216,6 +222,9 @@ def asc(col: "ColumnOrName") -> Column:
.. versionadded:: 1.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -266,6 +275,9 @@ def desc(col: "ColumnOrName") -> Column:
.. versionadded:: 1.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -301,6 +313,9 @@ def sqrt(col: "ColumnOrName") -> Column:
.. versionadded:: 1.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -331,6 +346,9 @@ def abs(col: "ColumnOrName") -> Column:
.. versionadded:: 1.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -361,6 +379,9 @@ def mode(col: "ColumnOrName") -> Column:
.. versionadded:: 3.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -396,6 +417,9 @@ def max(col: "ColumnOrName") -> Column:
.. versionadded:: 1.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -426,6 +450,9 @@ def min(col: "ColumnOrName") -> Column:
.. versionadded:: 1.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -456,6 +483,9 @@ def max_by(col: "ColumnOrName", ord: "ColumnOrName") -> Column:
.. versionadded:: 3.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -492,6 +522,9 @@ def min_by(col: "ColumnOrName", ord: "ColumnOrName") -> Column:
.. versionadded:: 3.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -528,6 +561,9 @@ def count(col: "ColumnOrName") -> Column:
.. versionadded:: 1.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -560,6 +596,9 @@ def sum(col: "ColumnOrName") -> Column:
.. versionadded:: 1.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -590,6 +629,9 @@ def avg(col: "ColumnOrName") -> Column:
.. versionadded:: 1.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -621,6 +663,9 @@ def mean(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -651,6 +696,9 @@ def median(col: "ColumnOrName") -> Column:
.. versionadded:: 3.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -686,6 +734,9 @@ def sumDistinct(col: "ColumnOrName") -> Column:
.. versionadded:: 1.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
.. deprecated:: 3.2.0
Use :func:`sum_distinct` instead.
"""
@@ -700,6 +751,9 @@ def sum_distinct(col: "ColumnOrName") -> Column:
.. versionadded:: 3.2.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -730,6 +784,9 @@ def product(col: "ColumnOrName") -> Column:
.. versionadded:: 3.2.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : str, :class:`Column`
@@ -763,6 +820,9 @@ def acos(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -794,6 +854,9 @@ def acosh(col: "ColumnOrName") -> Column:
.. versionadded:: 3.1.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -825,6 +888,9 @@ def asin(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -856,6 +922,9 @@ def asinh(col: "ColumnOrName") -> Column:
.. versionadded:: 3.1.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -886,6 +955,9 @@ def atan(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -916,6 +988,9 @@ def atanh(col: "ColumnOrName") -> Column:
.. versionadded:: 3.1.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -947,6 +1022,9 @@ def cbrt(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -977,6 +1055,9 @@ def ceil(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1007,6 +1088,9 @@ def cos(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1034,6 +1118,9 @@ def cosh(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1060,6 +1147,9 @@ def cot(col: "ColumnOrName") -> Column:
.. versionadded:: 3.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1087,6 +1177,9 @@ def csc(col: "ColumnOrName") -> Column:
.. versionadded:: 3.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1114,6 +1207,9 @@ def exp(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1144,6 +1240,9 @@ def expm1(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1170,6 +1269,9 @@ def floor(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1200,6 +1302,9 @@ def log(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1227,6 +1332,9 @@ def log10(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1257,6 +1365,9 @@ def log1p(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1290,6 +1401,9 @@ def rint(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1327,6 +1441,9 @@ def sec(col: "ColumnOrName") -> Column:
.. versionadded:: 3.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1353,6 +1470,9 @@ def signum(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1390,6 +1510,9 @@ def sin(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1417,6 +1540,9 @@ def sinh(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1444,6 +1570,9 @@ def tan(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1471,6 +1600,9 @@ def tanh(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1497,6 +1629,9 @@ def toDegrees(col: "ColumnOrName") -> Column:
"""
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
.. deprecated:: 2.1.0
Use :func:`degrees` instead.
"""
@@ -1509,6 +1644,9 @@ def toRadians(col: "ColumnOrName") -> Column:
"""
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
.. deprecated:: 2.1.0
Use :func:`radians` instead.
"""
@@ -1523,6 +1661,9 @@ def bitwiseNOT(col: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
.. deprecated:: 3.2.0
Use :func:`bitwise_not` instead.
"""
@@ -1537,6 +1678,9 @@ def bitwise_not(col: "ColumnOrName") -> Column:
.. versionadded:: 3.2.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1574,6 +1718,9 @@ def asc_nulls_first(col: "ColumnOrName") -> Column:
.. versionadded:: 2.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1614,6 +1761,9 @@ def asc_nulls_last(col: "ColumnOrName") -> Column:
.. versionadded:: 2.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1652,6 +1802,9 @@ def desc_nulls_first(col: "ColumnOrName") -> Column:
.. versionadded:: 2.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1692,6 +1845,9 @@ def desc_nulls_last(col: "ColumnOrName") -> Column:
.. versionadded:: 2.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1731,6 +1887,9 @@ def stddev(col: "ColumnOrName") -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1758,6 +1917,9 @@ def stddev_samp(col: "ColumnOrName") -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1785,6 +1947,9 @@ def stddev_pop(col: "ColumnOrName") -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1811,6 +1976,9 @@ def variance(col: "ColumnOrName") -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1842,6 +2010,9 @@ def var_samp(col: "ColumnOrName") -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1872,6 +2043,9 @@ def var_pop(col: "ColumnOrName") -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1898,6 +2072,9 @@ def skewness(col: "ColumnOrName") -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1924,6 +2101,9 @@ def kurtosis(col: "ColumnOrName") -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -1954,6 +2134,9 @@ def collect_list(col: "ColumnOrName") -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Notes
-----
The function is non-deterministic because the order of collected results depends
@@ -1985,6 +2168,9 @@ def collect_set(col: "ColumnOrName") -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Notes
-----
The function is non-deterministic because the order of collected results depends
@@ -2017,6 +2203,9 @@ def degrees(col: "ColumnOrName") -> Column:
.. versionadded:: 2.1.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -2045,6 +2234,9 @@ def radians(col: "ColumnOrName") -> Column:
.. versionadded:: 2.1.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -2069,6 +2261,9 @@ def atan2(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]
"""
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col1 : str, :class:`~pyspark.sql.Column` or float
@@ -2101,6 +2296,9 @@ def hypot(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col1 : str, :class:`~pyspark.sql.Column` or float
@@ -2129,6 +2327,9 @@ def pow(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float])
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col1 : str, :class:`~pyspark.sql.Column` or float
@@ -2157,6 +2358,9 @@ def pmod(dividend: Union["ColumnOrName", float], divisor: Union["ColumnOrName",
.. versionadded:: 3.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
dividend : str, :class:`~pyspark.sql.Column` or float
@@ -2202,6 +2406,9 @@ def row_number() -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Returns
-------
:class:`~pyspark.sql.Column`
@@ -2239,6 +2446,9 @@ def dense_rank() -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Returns
-------
:class:`~pyspark.sql.Column`
@@ -2279,6 +2489,9 @@ def rank() -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Returns
-------
:class:`~pyspark.sql.Column`
@@ -2312,6 +2525,9 @@ def cume_dist() -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Returns
-------
:class:`~pyspark.sql.Column`
@@ -2343,6 +2559,9 @@ def percent_rank() -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Returns
-------
:class:`~pyspark.sql.Column`
@@ -2373,6 +2592,9 @@ def approxCountDistinct(col: "ColumnOrName", rsd: Optional[float] = None) -> Col
"""
.. versionadded:: 1.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
.. deprecated:: 2.1.0
Use :func:`approx_count_distinct` instead.
"""
@@ -2387,6 +2609,12 @@ def approx_count_distinct(col: "ColumnOrName", rsd: Optional[float] = None) -> C
.. versionadded:: 2.1.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -2422,6 +2650,9 @@ def broadcast(df: DataFrame) -> DataFrame:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Returns
-------
:class:`~pyspark.sql.DataFrame`
@@ -2453,6 +2684,9 @@ def coalesce(*cols: "ColumnOrName") -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
cols : :class:`~pyspark.sql.Column` or str
@@ -2503,6 +2737,9 @@ def corr(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col1 : :class:`~pyspark.sql.Column` or str
@@ -2533,6 +2770,9 @@ def covar_pop(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col1 : :class:`~pyspark.sql.Column` or str
@@ -2563,6 +2803,9 @@ def covar_samp(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col1 : :class:`~pyspark.sql.Column` or str
@@ -2594,6 +2837,9 @@ def countDistinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column:
directly.
.. versionadded:: 1.3.0
+
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
"""
return count_distinct(col, *cols)
@@ -2604,6 +2850,9 @@ def count_distinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column:
.. versionadded:: 3.2.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -2655,6 +2904,9 @@ def first(col: "ColumnOrName", ignorenulls: bool = False) -> Column:
.. versionadded:: 1.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Notes
-----
The function is non-deterministic because its results depends on the order of the
@@ -2705,6 +2957,9 @@ def grouping(col: "ColumnOrName") -> Column:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -2739,6 +2994,9 @@ def grouping_id(*cols: "ColumnOrName") -> Column:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Notes
-----
The list of columns should match with grouping columns exactly, or empty (means all
@@ -2782,6 +3040,9 @@ def input_file_name() -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Returns
-------
:class:`~pyspark.sql.Column`
@@ -2804,6 +3065,9 @@ def isnan(col: "ColumnOrName") -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -2834,6 +3098,9 @@ def isnull(col: "ColumnOrName") -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -2867,6 +3134,9 @@ def last(col: "ColumnOrName", ignorenulls: bool = False) -> Column:
.. versionadded:: 1.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Notes
-----
The function is non-deterministic because its results depends on the order of the
@@ -2920,6 +3190,9 @@ def monotonically_increasing_id() -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Notes
-----
The function is non-deterministic because its result depends on partition IDs.
@@ -2950,6 +3223,9 @@ def nanvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col1 : :class:`~pyspark.sql.Column` or str
@@ -2984,6 +3260,9 @@ def percentile_approx(
.. versionadded:: 3.1.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -3053,6 +3332,9 @@ def rand(seed: Optional[int] = None) -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Notes
-----
The function is non-deterministic in general case.
@@ -3091,6 +3373,9 @@ def randn(seed: Optional[int] = None) -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Notes
-----
The function is non-deterministic in general case.
@@ -3130,6 +3415,9 @@ def round(col: "ColumnOrName", scale: int = 0) -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -3158,6 +3446,9 @@ def bround(col: "ColumnOrName", scale: int = 0) -> Column:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -3184,6 +3475,9 @@ def shiftLeft(col: "ColumnOrName", numBits: int) -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
.. deprecated:: 3.2.0
Use :func:`shiftleft` instead.
"""
@@ -3197,6 +3491,9 @@ def shiftleft(col: "ColumnOrName", numBits: int) -> Column:
.. versionadded:: 3.2.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -3223,6 +3520,9 @@ def shiftRight(col: "ColumnOrName", numBits: int) -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
.. deprecated:: 3.2.0
Use :func:`shiftright` instead.
"""
@@ -3236,6 +3536,9 @@ def shiftright(col: "ColumnOrName", numBits: int) -> Column:
.. versionadded:: 3.2.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -3262,6 +3565,9 @@ def shiftRightUnsigned(col: "ColumnOrName", numBits: int) -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
.. deprecated:: 3.2.0
Use :func:`shiftrightunsigned` instead.
"""
@@ -3275,6 +3581,9 @@ def shiftrightunsigned(col: "ColumnOrName", numBits: int) -> Column:
.. versionadded:: 3.2.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -3302,6 +3611,9 @@ def spark_partition_id() -> Column:
.. versionadded:: 1.6.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Notes
-----
This is non deterministic because it depends on data partitioning and task scheduling.
@@ -3326,6 +3638,9 @@ def expr(str: str) -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
str : str
@@ -3368,6 +3683,9 @@ def struct(
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
cols : list, set, str or :class:`~pyspark.sql.Column`
@@ -3399,6 +3717,9 @@ def greatest(*cols: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -3428,6 +3749,9 @@ def least(*cols: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
cols : :class:`~pyspark.sql.Column` or str
@@ -3457,6 +3781,9 @@ def when(condition: Column, value: Any) -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
condition : :class:`~pyspark.sql.Column`
@@ -3516,6 +3843,9 @@ def log(arg1: Union["ColumnOrName", float], arg2: Optional["ColumnOrName"] = Non
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
arg1 : :class:`~pyspark.sql.Column`, str or float
@@ -3563,6 +3893,9 @@ def log2(col: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -3593,6 +3926,9 @@ def conv(col: "ColumnOrName", fromBase: int, toBase: int) -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -3623,6 +3959,9 @@ def factorial(col: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -3656,6 +3995,9 @@ def lag(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) ->
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -3734,6 +4076,9 @@ def lead(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) ->
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -3814,6 +4159,9 @@ def nth_value(col: "ColumnOrName", offset: int, ignoreNulls: Optional[bool] = Fa
.. versionadded:: 3.1.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -3884,6 +4232,9 @@ def ntile(n: int) -> Column:
.. versionadded:: 1.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
n : int
@@ -3938,6 +4289,9 @@ def current_date() -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Returns
-------
:class:`~pyspark.sql.Column`
@@ -3964,6 +4318,9 @@ def current_timestamp() -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Returns
-------
:class:`~pyspark.sql.Column`
@@ -3991,6 +4348,9 @@ def localtimestamp() -> Column:
.. versionadded:: 3.4.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Returns
-------
:class:`~pyspark.sql.Column`
@@ -4022,6 +4382,9 @@ def date_format(date: "ColumnOrName", format: str) -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Notes
-----
Whenever possible, use specialized functions like `year`.
@@ -4054,6 +4417,9 @@ def year(col: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -4080,6 +4446,9 @@ def quarter(col: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -4106,6 +4475,9 @@ def month(col: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -4133,6 +4505,9 @@ def dayofweek(col: "ColumnOrName") -> Column:
.. versionadded:: 2.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -4159,6 +4534,9 @@ def dayofmonth(col: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -4185,6 +4563,9 @@ def dayofyear(col: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -4211,6 +4592,9 @@ def hour(col: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -4238,6 +4622,9 @@ def minute(col: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -4265,6 +4652,9 @@ def second(col: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -4294,6 +4684,9 @@ def weekofyear(col: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -4320,6 +4713,9 @@ def make_date(year: "ColumnOrName", month: "ColumnOrName", day: "ColumnOrName")
.. versionadded:: 3.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
year : :class:`~pyspark.sql.Column` or str
@@ -4351,6 +4747,9 @@ def date_add(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
start : :class:`~pyspark.sql.Column` or str
@@ -4386,6 +4785,9 @@ def date_sub(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
start : :class:`~pyspark.sql.Column` or str
@@ -4420,6 +4822,9 @@ def datediff(end: "ColumnOrName", start: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
end : :class:`~pyspark.sql.Column` or str
@@ -4449,6 +4854,9 @@ def add_months(start: "ColumnOrName", months: Union["ColumnOrName", int]) -> Col
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
start : :class:`~pyspark.sql.Column` or str
@@ -4525,6 +4933,9 @@ def to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column:
.. versionadded:: 2.2.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -4574,6 +4985,9 @@ def to_timestamp(col: "ColumnOrName", format: Optional[str] = None) -> Column:
.. versionadded:: 2.2.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -4609,6 +5023,9 @@ def trunc(date: "ColumnOrName", format: str) -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
date : :class:`~pyspark.sql.Column` or str
@@ -4641,6 +5058,9 @@ def date_trunc(format: str, timestamp: "ColumnOrName") -> Column:
.. versionadded:: 2.3.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
format : str
@@ -4676,6 +5096,9 @@ def next_day(date: "ColumnOrName", dayOfWeek: str) -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
date : :class:`~pyspark.sql.Column` or str
@@ -4705,6 +5128,9 @@ def last_day(date: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
date : :class:`~pyspark.sql.Column` or str
@@ -4733,6 +5159,9 @@ def from_unixtime(timestamp: "ColumnOrName", format: str = "yyyy-MM-dd HH:mm:ss"
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
timestamp : :class:`~pyspark.sql.Column` or str
@@ -4779,6 +5208,9 @@ def unix_timestamp(
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
timestamp : :class:`~pyspark.sql.Column` or str, optional
@@ -4822,6 +5254,9 @@ def from_utc_timestamp(timestamp: "ColumnOrName", tz: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
timestamp : :class:`~pyspark.sql.Column` or str
@@ -4873,6 +5308,9 @@ def to_utc_timestamp(timestamp: "ColumnOrName", tz: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
timestamp : :class:`~pyspark.sql.Column` or str
@@ -4914,6 +5352,9 @@ def timestamp_seconds(col: "ColumnOrName") -> Column:
.. versionadded:: 3.1.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -5146,6 +5587,9 @@ def crc32(col: "ColumnOrName") -> Column:
Calculates the cyclic redundancy check value (CRC32) of a binary column and
returns the value as a bigint.
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -5172,6 +5616,9 @@ def md5(col: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -5196,6 +5643,9 @@ def sha1(col: "ColumnOrName") -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
+
Parameters
----------
col : :class:`~pyspark.sql.Column` or str
@@ -5222,6 +5672,9 @@ def sha2(col: "ColumnOrName", numBits: int) -> Column:
.. versionadded:: 1.5.0
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
... 977 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org