You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/06/17 18:34:50 UTC
(datafusion-python) branch asf-staging updated: Publish built docs triggered by 2d074713d1505256ed92a07e77c18ee43713a303
This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a commit to branch asf-staging
in repository https://gitbox.apache.org/repos/asf/datafusion-python.git
The following commit(s) were added to refs/heads/asf-staging by this push:
new 3f71547b Publish built docs triggered by 2d074713d1505256ed92a07e77c18ee43713a303
3f71547b is described below
commit 3f71547b09677cd2e40cfd37296b4fd33f38cf2b
Author: github-actions[bot] <gi...@users.noreply.github.com>
AuthorDate: Wed Jun 17 18:34:44 2026 +0000
Publish built docs triggered by 2d074713d1505256ed92a07e77c18ee43713a303
---
_sources/autoapi/datafusion/context/index.rst.txt | 24 +
.../autoapi/datafusion/functions/index.rst.txt | 13 +-
.../datafusion/functions/spark/index.rst.txt | 1571 +++++++++++++
.../common-operations/aggregations.md.txt | 10 +
.../user-guide/common-operations/functions.md.txt | 6 +
_sources/user-guide/common-operations/index.md.txt | 1 +
.../common-operations/spark-functions.md.txt | 91 +
autoapi/datafusion/catalog/index.html | 6 +-
autoapi/datafusion/context/index.html | 29 +-
autoapi/datafusion/dataframe/index.html | 6 +-
autoapi/datafusion/dataframe_formatter/index.html | 6 +-
autoapi/datafusion/expr/index.html | 6 +-
autoapi/datafusion/functions/index.html | 27 +-
autoapi/datafusion/functions/spark/index.html | 2458 ++++++++++++++++++++
autoapi/datafusion/index.html | 6 +-
autoapi/datafusion/input/base/index.html | 6 +-
autoapi/datafusion/input/index.html | 12 +-
autoapi/datafusion/input/location/index.html | 6 +-
autoapi/datafusion/io/index.html | 6 +-
autoapi/datafusion/ipc/index.html | 6 +-
autoapi/datafusion/object_store/index.html | 6 +-
autoapi/datafusion/options/index.html | 6 +-
autoapi/datafusion/plan/index.html | 6 +-
autoapi/datafusion/record_batch/index.html | 6 +-
autoapi/datafusion/substrait/index.html | 6 +-
autoapi/datafusion/unparser/index.html | 6 +-
autoapi/datafusion/user_defined/index.html | 6 +-
autoapi/index.html | 11 +-
contributor-guide/ffi.html | 6 +-
contributor-guide/index.html | 6 +-
contributor-guide/introduction.html | 6 +-
genindex.html | 260 ++-
index.html | 6 +-
links.html | 6 +-
objects.inv | Bin 11106 -> 11697 bytes
py-modindex.html | 11 +-
search.html | 6 +-
searchindex.js | 2 +-
user-guide/ai-coding-assistants.html | 6 +-
user-guide/basics.html | 6 +-
user-guide/common-operations/aggregations.html | 46 +-
user-guide/common-operations/basic-info.html | 40 +-
user-guide/common-operations/expressions.html | 6 +-
user-guide/common-operations/functions.html | 38 +-
user-guide/common-operations/index.html | 13 +-
user-guide/common-operations/joins.html | 6 +-
.../common-operations/select-and-filter.html | 6 +-
.../{views.html => spark-functions.html} | 138 +-
user-guide/common-operations/udf-and-udfa.html | 10 +-
user-guide/common-operations/views.html | 6 +-
user-guide/common-operations/windows.html | 6 +-
user-guide/configuration.html | 6 +-
user-guide/data-sources.html | 6 +-
user-guide/dataframe/execution-metrics.html | 6 +-
user-guide/dataframe/index.html | 6 +-
user-guide/dataframe/rendering.html | 6 +-
user-guide/distributing-work.html | 6 +-
user-guide/index.html | 7 +-
user-guide/introduction.html | 6 +-
user-guide/io/arrow.html | 6 +-
user-guide/io/avro.html | 6 +-
user-guide/io/csv.html | 6 +-
user-guide/io/index.html | 6 +-
user-guide/io/json.html | 6 +-
user-guide/io/parquet.html | 6 +-
user-guide/io/table_provider.html | 6 +-
user-guide/sql.html | 6 +-
user-guide/upgrade-guides.html | 6 +-
68 files changed, 4913 insertions(+), 175 deletions(-)
diff --git a/_sources/autoapi/datafusion/context/index.rst.txt b/_sources/autoapi/datafusion/context/index.rst.txt
index 9f3d1533..ad379f02 100644
--- a/_sources/autoapi/datafusion/context/index.rst.txt
+++ b/_sources/autoapi/datafusion/context/index.rst.txt
@@ -631,6 +631,30 @@ Module Contents
+ .. py:method:: enable_spark_functions() -> None
+
+ Register all Spark-compatible functions for SQL access.
+
+ Registers every UDF/UDAF/UDWF from the ``datafusion-spark`` crate,
+ overriding any DataFusion built-ins of the same name with their
+ Spark-semantics version (e.g. ``substring`` becomes 1-indexed,
+ ``concat`` propagates NULL, ``round`` uses HALF_UP rounding).
+
+ For DataFrame use, import the typed wrappers from
+ :py:mod:`datafusion.functions.spark` directly; this method is only
+ needed for SQL queries.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> ctx.enable_spark_functions()
+ >>> ctx.sql(
+ ... "SELECT sha2('hello', 256) AS h"
+ ... ).collect_column("h")[0].as_py()
+ '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824'
+
+
+
.. py:method:: enable_url_table() -> SessionContext
Control if local files can be queried as tables.
diff --git a/_sources/autoapi/datafusion/functions/index.rst.txt b/_sources/autoapi/datafusion/functions/index.rst.txt
index 8d20ff96..22e7a00a 100644
--- a/_sources/autoapi/datafusion/functions/index.rst.txt
+++ b/_sources/autoapi/datafusion/functions/index.rst.txt
@@ -29,6 +29,15 @@ datafusion.functions
+Submodules
+----------
+
+.. toctree::
+ :maxdepth: 1
+
+ /autoapi/datafusion/functions/spark/index
+
+
Attributes
----------
@@ -362,8 +371,8 @@ Functions
datafusion.functions.with_metadata
-Module Contents
----------------
+Package Contents
+----------------
.. py:function:: abs(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
diff --git a/_sources/autoapi/datafusion/functions/spark/index.rst.txt b/_sources/autoapi/datafusion/functions/spark/index.rst.txt
new file mode 100644
index 00000000..d69eaa7d
--- /dev/null
+++ b/_sources/autoapi/datafusion/functions/spark/index.rst.txt
@@ -0,0 +1,1571 @@
+datafusion.functions.spark
+==========================
+
+.. py:module:: datafusion.functions.spark
+
+.. autoapi-nested-parse::
+
+ Spark-compatible function bindings.
+
+ These functions mirror the semantics of their Apache Spark counterparts
+ exactly. Some override DataFusion built-ins (``substring`` is 1-indexed,
+ ``concat`` propagates NULL, ``round`` uses HALF_UP rounding, etc.), which is
+ why they live in a separate namespace rather than replacing the defaults.
+
+ For DataFrame use, import this module and call functions directly. For SQL
+ use, call :py:meth:`datafusion.SessionContext.enable_spark_functions` to
+ register the Spark UDFs by name (overriding any built-ins with matching
+ names) before issuing SQL queries.
+
+
+
+Functions
+---------
+
+.. autoapisummary::
+
+ datafusion.functions.spark.abs
+ datafusion.functions.spark.add_months
+ datafusion.functions.spark.array
+ datafusion.functions.spark.array_contains
+ datafusion.functions.spark.array_repeat
+ datafusion.functions.spark.ascii
+ datafusion.functions.spark.avg
+ datafusion.functions.spark.base64
+ datafusion.functions.spark.bin
+ datafusion.functions.spark.bit_count
+ datafusion.functions.spark.bit_get
+ datafusion.functions.spark.bitmap_bit_position
+ datafusion.functions.spark.bitmap_bucket_number
+ datafusion.functions.spark.bitmap_count
+ datafusion.functions.spark.bitwise_not
+ datafusion.functions.spark.ceil
+ datafusion.functions.spark.char
+ datafusion.functions.spark.collect_list
+ datafusion.functions.spark.collect_set
+ datafusion.functions.spark.concat
+ datafusion.functions.spark.crc32
+ datafusion.functions.spark.csc
+ datafusion.functions.spark.date_add
+ datafusion.functions.spark.date_diff
+ datafusion.functions.spark.date_part
+ datafusion.functions.spark.date_sub
+ datafusion.functions.spark.date_trunc
+ datafusion.functions.spark.elt
+ datafusion.functions.spark.expm1
+ datafusion.functions.spark.factorial
+ datafusion.functions.spark.floor
+ datafusion.functions.spark.format_string
+ datafusion.functions.spark.from_utc_timestamp
+ datafusion.functions.spark.hex
+ datafusion.functions.spark.hour
+ datafusion.functions.spark.if_
+ datafusion.functions.spark.ilike
+ datafusion.functions.spark.is_valid_utf8
+ datafusion.functions.spark.json_tuple
+ datafusion.functions.spark.last_day
+ datafusion.functions.spark.length
+ datafusion.functions.spark.like
+ datafusion.functions.spark.luhn_check
+ datafusion.functions.spark.make_dt_interval
+ datafusion.functions.spark.make_interval
+ datafusion.functions.spark.make_valid_utf8
+ datafusion.functions.spark.map_from_arrays
+ datafusion.functions.spark.map_from_entries
+ datafusion.functions.spark.minute
+ datafusion.functions.spark.modulus
+ datafusion.functions.spark.negative
+ datafusion.functions.spark.next_day
+ datafusion.functions.spark.parse_url
+ datafusion.functions.spark.pmod
+ datafusion.functions.spark.rint
+ datafusion.functions.spark.round
+ datafusion.functions.spark.sec
+ datafusion.functions.spark.second
+ datafusion.functions.spark.sha1
+ datafusion.functions.spark.sha2
+ datafusion.functions.spark.shiftleft
+ datafusion.functions.spark.shiftright
+ datafusion.functions.spark.shiftrightunsigned
+ datafusion.functions.spark.shuffle
+ datafusion.functions.spark.size
+ datafusion.functions.spark.slice
+ datafusion.functions.spark.soundex
+ datafusion.functions.spark.space
+ datafusion.functions.spark.spark_cast
+ datafusion.functions.spark.str_to_map
+ datafusion.functions.spark.substring
+ datafusion.functions.spark.time_trunc
+ datafusion.functions.spark.to_utc_timestamp
+ datafusion.functions.spark.trunc
+ datafusion.functions.spark.try_parse_url
+ datafusion.functions.spark.try_sum
+ datafusion.functions.spark.try_url_decode
+ datafusion.functions.spark.unbase64
+ datafusion.functions.spark.unhex
+ datafusion.functions.spark.unix_date
+ datafusion.functions.spark.unix_micros
+ datafusion.functions.spark.unix_millis
+ datafusion.functions.spark.unix_seconds
+ datafusion.functions.spark.url_decode
+ datafusion.functions.spark.url_encode
+ datafusion.functions.spark.width_bucket
+ datafusion.functions.spark.xxhash64
+
+
+Module Contents
+---------------
+
+.. py:function:: abs(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``abs``: absolute value.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.abs(dfn.lit(-5)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 5
+
+
+.. py:function:: add_months(start: datafusion.expr.Expr, months: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+
+ Spark ``add_months``: date + N months.
+
+ ``months`` accepts a native ``int`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import date
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> d = dfn.lit(pa.scalar(date(2020, 1, 15), type=pa.date32()))
+ >>> r = df.select(dfn.functions.spark.add_months(d, 2).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ datetime.date(2020, 3, 15)
+
+
+.. py:function:: array(*cols: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``array``: builds an array from the given elements.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.array(
+ ... dfn.lit(1), dfn.lit(2), dfn.lit(3)
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ [1, 2, 3]
+
+
+.. py:function:: array_contains(col: datafusion.expr.Expr, value: datafusion.expr.Expr | Any) -> datafusion.expr.Expr
+
+ Spark ``array_contains``: true if the array contains the element.
+
+ ``value`` accepts a native Python literal or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.array_contains(
+ ... dfn.functions.spark.array(dfn.lit(1), dfn.lit(2)), 1
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ True
+
+
+.. py:function:: array_repeat(col: datafusion.expr.Expr, count: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+
+ Spark ``array_repeat``: array of ``element`` repeated ``count`` times.
+
+ ``count`` accepts a native ``int`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.array_repeat(dfn.lit("a"), 3).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ ['a', 'a', 'a']
+
+
+.. py:function:: ascii(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``ascii``: code point of the first character.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.ascii(dfn.lit("A")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 65
+
+
+.. py:function:: avg(col: datafusion.expr.Expr, distinct: bool | None = None, filter: datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None, null_treatment: datafusion.common.NullTreatment | None = None) -> datafusion.expr.Expr
+
+ Spark ``avg``: returns the mean of a numeric column.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0]})
+ >>> r = df.aggregate(
+ ... [], [dfn.functions.spark.avg(dfn.col("a")).alias("v")])
+ >>> r.collect_column("v")[0].as_py()
+ 2.0
+
+
+.. py:function:: base64(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``base64``: encode binary as a base64 string.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.base64(dfn.lit(b"hi")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 'aGk='
+
+
+.. py:function:: bin(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``bin``: binary string representation of a long.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.bin(dfn.lit(7)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ '111'
+
+
+.. py:function:: bit_count(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``bit_count``: number of bits set in the integer's binary form.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.bit_count(dfn.lit(7)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 3
+
+
+.. py:function:: bit_get(col: datafusion.expr.Expr, pos: datafusion.expr.Expr | str) -> datafusion.expr.Expr
+
+ Spark ``bit_get``: returns the bit (0 or 1) at ``pos``.
+
+ A bare ``str`` ``pos`` is treated as a column name (matching pyspark),
+ not a literal; pass :func:`~datafusion.lit` for a literal position.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.bit_get(dfn.lit(5), dfn.lit(0)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 1
+
+
+.. py:function:: bitmap_bit_position(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``bitmap_bit_position``: bit position for a child expression.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.bitmap_bit_position(dfn.lit(15)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 14
+
+
+.. py:function:: bitmap_bucket_number(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``bitmap_bucket_number``: bucket number for a child expression.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.bitmap_bucket_number(dfn.lit(15)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 1
+
+
+.. py:function:: bitmap_count(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``bitmap_count``: number of set bits in a bitmap.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.bitmap_count(dfn.lit(b"\xff")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 8
+
+
+.. py:function:: bitwise_not(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``~``: bitwise NOT.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.bitwise_not(dfn.lit(0)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ -1
+
+
+.. py:function:: ceil(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``ceil``: smallest integer ≥ arg.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.ceil(dfn.lit(1.2)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 2
+
+
+.. py:function:: char(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``char``: ASCII character for a code point (mod 256).
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.char(dfn.lit(65)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 'A'
+
+
+.. py:function:: collect_list(col: datafusion.expr.Expr, distinct: bool | None = None, filter: datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None, null_treatment: datafusion.common.NullTreatment | None = None) -> datafusion.expr.Expr
+
+ Spark ``collect_list``: collect values into an array (preserves dups).
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 2]})
+ >>> r = df.aggregate(
+ ... [], [dfn.functions.spark.collect_list(dfn.col("a")).alias("v")])
+ >>> sorted(r.collect_column("v")[0].as_py())
+ [1, 2, 2]
+
+
+.. py:function:: collect_set(col: datafusion.expr.Expr, distinct: bool | None = None, filter: datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None, null_treatment: datafusion.common.NullTreatment | None = None) -> datafusion.expr.Expr
+
+ Spark ``collect_set``: collect distinct values into an array.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 2, 3]})
+ >>> r = df.aggregate(
+ ... [], [dfn.functions.spark.collect_set(dfn.col("a")).alias("v")])
+ >>> sorted(r.collect_column("v")[0].as_py())
+ [1, 2, 3]
+
+
+.. py:function:: concat(*cols: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``concat``: concatenates strings; NULL if any input is NULL.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.concat(dfn.lit("a"), dfn.lit("b")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 'ab'
+
+
+.. py:function:: crc32(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``crc32``: cyclic redundancy check value as a bigint.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"s": ["ABC"]})
+ >>> r = df.select(dfn.functions.spark.crc32(dfn.col("s")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 2743272264
+
+
+.. py:function:: csc(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``csc``: cosecant.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.csc(dfn.lit(1.5708)).alias("v"))
+ >>> f"{r.collect_column('v')[0].as_py():.4f}"
+ '1.0000'
+
+
+.. py:function:: date_add(start: datafusion.expr.Expr, days: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+
+ Spark ``date_add``: date + N days.
+
+ ``days`` accepts a native ``int`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import date
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> d = dfn.lit(pa.scalar(date(2020, 1, 15), type=pa.date32()))
+ >>> r = df.select(dfn.functions.spark.date_add(d, 5).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ datetime.date(2020, 1, 20)
+
+
+.. py:function:: date_diff(end: datafusion.expr.Expr, start: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``date_diff``: number of days from ``start_date`` to ``end_date``.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import date
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> d = dfn.lit(pa.scalar(date(2020, 1, 15), type=pa.date32()))
+ >>> end = dfn.lit(pa.scalar(date(2020, 1, 20), type=pa.date32()))
+ >>> r = df.select(dfn.functions.spark.date_diff(end, d).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 5
+
+
+.. py:function:: date_part(field: datafusion.expr.Expr | str, source: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``date_part``: extract ``field`` from a date/time/timestamp.
+
+ ``field`` accepts a native ``str`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import date
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> d = dfn.lit(pa.scalar(date(2020, 1, 15), type=pa.date32()))
+ >>> r = df.select(
+ ... dfn.functions.spark.date_part("year", d).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 2020
+
+
+.. py:function:: date_sub(start: datafusion.expr.Expr, days: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+
+ Spark ``date_sub``: date - N days.
+
+ ``days`` accepts a native ``int`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import date
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> d = dfn.lit(pa.scalar(date(2020, 1, 15), type=pa.date32()))
+ >>> r = df.select(dfn.functions.spark.date_sub(d, 5).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ datetime.date(2020, 1, 10)
+
+
+.. py:function:: date_trunc(format: datafusion.expr.Expr | str, timestamp: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``date_trunc``: truncate timestamp to unit ``fmt``.
+
+ ``format`` accepts a native ``str`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import datetime
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> ts = dfn.lit(
+ ... pa.scalar(datetime(2020, 1, 15, 14, 30, 45),
+ ... type=pa.timestamp('us')))
+ >>> r = df.select(
+ ... dfn.functions.spark.date_trunc("month", ts).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ datetime.datetime(2020, 1, 1, 0, 0)
+
+
+.. py:function:: elt(*inputs: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``elt``: returns the n-th input (1-indexed).
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.elt(
+ ... dfn.lit(2), dfn.lit("a"), dfn.lit("b")
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ 'b'
+
+
+.. py:function:: expm1(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``expm1``: exp(arg) - 1.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.expm1(dfn.lit(0.0)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 0.0
+
+
+.. py:function:: factorial(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``factorial``: n! for n in [0..20], else NULL.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.factorial(
+ ... dfn.lit(pa.scalar(5, type=pa.int32()))
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ 120
+
+
+.. py:function:: floor(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``floor``: largest integer ≤ arg.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.floor(dfn.lit(1.8)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 1
+
+
+.. py:function:: format_string(format: str | datafusion.expr.Expr, *cols: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``format_string``: printf-style format string.
+
+ ``format`` is the printf-style template (a plain ``str`` is auto-promoted
+ to a literal expression); remaining args are values to substitute.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.format_string(
+ ... "%d-%s", dfn.lit(42), dfn.lit("hi")
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ '42-hi'
+
+
+.. py:function:: from_utc_timestamp(timestamp: datafusion.expr.Expr, tz: datafusion.expr.Expr | str) -> datafusion.expr.Expr
+
+ Spark ``from_utc_timestamp``: interpret ``ts`` as UTC, convert to ``tz``.
+
+ ``tz`` accepts a native ``str`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import datetime
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> ts = dfn.lit(
+ ... pa.scalar(datetime(2020, 1, 15, 14, 30, 45),
+ ... type=pa.timestamp('us')))
+ >>> r = df.select(
+ ... dfn.functions.spark.from_utc_timestamp(ts, "UTC").alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ datetime.datetime(2020, 1, 15, 14, 30, 45)
+
+
+.. py:function:: hex(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``hex``: hexadecimal representation.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.hex(dfn.lit(255)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 'FF'
+
+
+.. py:function:: hour(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``hour``: extract hour component of a timestamp.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import datetime
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> ts = dfn.lit(
+ ... pa.scalar(datetime(2020, 1, 15, 14, 30, 45),
+ ... type=pa.timestamp('us')))
+ >>> r = df.select(dfn.functions.spark.hour(ts).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 14
+
+
+.. py:function:: if_(condition: datafusion.expr.Expr, if_true: datafusion.expr.Expr | Any, if_false: datafusion.expr.Expr | Any) -> datafusion.expr.Expr
+
+ Spark ``if``: returns ``if_true`` when ``condition`` is true, else ``if_false``.
+
+ Exposed as ``if_`` because ``if`` is a Python keyword. ``if_true`` and
+ ``if_false`` accept native Python literals or :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.if_(
+ ... dfn.lit(2) > dfn.lit(1), "big", "small"
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ 'big'
+
+
+.. py:function:: ilike(str: datafusion.expr.Expr, pattern: datafusion.expr.Expr | ilike.str, escapeChar: ilike.str | None = None) -> datafusion.expr.Expr
+
+ Spark ``ilike``: case-insensitive pattern match.
+
+ A bare ``str`` ``pattern`` is treated as a column name (matching pyspark),
+ not a literal; pass :func:`~datafusion.lit` for a literal pattern.
+ ``escapeChar`` is accepted for pyspark parity but is not yet wired through
+ the Rust binding; passing a non-``None`` value raises ``NotImplementedError``.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.ilike(dfn.lit("HELLO"), dfn.lit("h%")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ True
+
+
+.. py:function:: is_valid_utf8(str: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``is_valid_utf8``: true if the string is valid UTF-8.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.is_valid_utf8(dfn.lit("hello")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ True
+
+
+.. py:function:: json_tuple(col: datafusion.expr.Expr, *fields: datafusion.expr.Expr | str) -> datafusion.expr.Expr
+
+ Spark ``json_tuple``: extract top-level fields from a JSON string.
+
+ Each field name accepts a native ``str`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.json_tuple(
+ ... dfn.lit('{"a":1,"b":"x"}'), "a", "b"
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ {'c0': '1', 'c1': 'x'}
+
+
+.. py:function:: last_day(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``last_day``: last day of the month containing the date.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import date
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> d = dfn.lit(pa.scalar(date(2020, 1, 15), type=pa.date32()))
+ >>> r = df.select(dfn.functions.spark.last_day(d).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ datetime.date(2020, 1, 31)
+
+
+.. py:function:: length(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``length``: character length of a string, or byte length of binary.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.length(dfn.lit("hello")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 5
+
+
+.. py:function:: like(str: datafusion.expr.Expr, pattern: datafusion.expr.Expr | like.str, escapeChar: like.str | None = None) -> datafusion.expr.Expr
+
+ Spark ``like``: case-sensitive pattern match.
+
+ A bare ``str`` ``pattern`` is treated as a column name (matching pyspark),
+ not a literal; pass :func:`~datafusion.lit` for a literal pattern.
+ ``escapeChar`` is accepted for pyspark parity but is not yet wired through
+ the Rust binding; passing a non-``None`` value raises ``NotImplementedError``.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.like(dfn.lit("hello"), dfn.lit("h%")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ True
+
+
+.. py:function:: luhn_check(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``luhn_check``: true if the digit string passes the Luhn check.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.luhn_check(
+ ... dfn.lit("4111111111111111")
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ True
+
+
+.. py:function:: make_dt_interval(days: datafusion.expr.Expr | int | None = None, hours: datafusion.expr.Expr | int | None = None, mins: datafusion.expr.Expr | int | None = None, secs: datafusion.expr.Expr | float | None = None) -> datafusion.expr.Expr
+
+ Spark ``make_dt_interval``: day-time interval from components.
+
+ All parts are optional; omitted parts default to zero, matching pyspark.
+ Integer parts accept a native ``int`` and ``secs`` accepts a ``float``,
+ or any part may be an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.make_dt_interval().alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ datetime.timedelta(0)
+
+ >>> r = df.select(
+ ... dfn.functions.spark.make_dt_interval(
+ ... days=1, hours=2, mins=3, secs=4.5
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ datetime.timedelta(days=1, seconds=7384, microseconds=500000)
+
+
+.. py:function:: make_interval(years: datafusion.expr.Expr | int | None = None, months: datafusion.expr.Expr | int | None = None, weeks: datafusion.expr.Expr | int | None = None, days: datafusion.expr.Expr | int | None = None, hours: datafusion.expr.Expr | int | None = None, mins: datafusion.expr.Expr | int | None = None, secs: datafusion.expr.Expr | float | None = None) -> datafusion.expr.Expr
+
+ Spark ``make_interval``: interval from year/month/week/day/hour/min/sec parts.
+
+ All parts are optional; omitted parts default to zero, matching pyspark.
+ Integer parts accept a native ``int`` and ``secs`` accepts a ``float``,
+ or any part may be an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.make_interval().alias("v"))
+ >>> r.collect_column("v")[0].as_py().months
+ 0
+
+ >>> r = df.select(dfn.functions.spark.make_interval(years=1).alias("v"))
+ >>> r.collect_column("v")[0].as_py().months
+ 12
+
+
+.. py:function:: make_valid_utf8(str: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``make_valid_utf8``: replace invalid UTF-8 bytes with U+FFFD.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.make_valid_utf8(dfn.lit("hello")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 'hello'
+
+
+.. py:function:: map_from_arrays(col1: datafusion.expr.Expr, col2: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``map_from_arrays``: build a map from parallel key/value arrays.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> keys = dfn.functions.spark.array(dfn.lit("a"), dfn.lit("b"))
+ >>> vals = dfn.functions.spark.array(dfn.lit(1), dfn.lit(2))
+ >>> r = df.select(
+ ... dfn.functions.spark.map_from_arrays(keys, vals).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ [('a', 1), ('b', 2)]
+
+
+.. py:function:: map_from_entries(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``map_from_entries``: build a map from an array of key/value structs.
+
+ ``col`` must be an array whose elements are two-field structs; the first
+ field becomes the map key and the second the value.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> ctx = dfn.SessionContext()
+ >>> entry_type = pa.list_(
+ ... pa.struct([("key", pa.string()), ("value", pa.int64())]))
+ >>> entries = pa.array(
+ ... [[{"key": "a", "value": 1}, {"key": "b", "value": 2}]],
+ ... type=entry_type)
+ >>> df = ctx.from_arrow(pa.record_batch([entries], names=["e"]))
+ >>> r = df.select(
+ ... dfn.functions.spark.map_from_entries(dfn.col("e")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ [('a', 1), ('b', 2)]
+
+
+.. py:function:: minute(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``minute``: extract minute component of a timestamp.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import datetime
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> ts = dfn.lit(
+ ... pa.scalar(datetime(2020, 1, 15, 14, 30, 45),
+ ... type=pa.timestamp('us')))
+ >>> r = df.select(dfn.functions.spark.minute(ts).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 30
+
+
+.. py:function:: modulus(dividend: datafusion.expr.Expr | float, divisor: datafusion.expr.Expr | float) -> datafusion.expr.Expr
+
+ Spark ``mod``: remainder of ``dividend / divisor`` (sign follows dividend).
+
+ ``dividend`` and ``divisor`` accept native numbers or :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.modulus(dfn.lit(10), dfn.lit(3)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 1
+
+
+.. py:function:: negative(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``negative``: unary minus.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.negative(dfn.lit(3)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ -3
+
+
+.. py:function:: next_day(date: datafusion.expr.Expr, dayOfWeek: datafusion.expr.Expr | str) -> datafusion.expr.Expr
+
+ Spark ``next_day``: first date after ``start_date`` named ``day_of_week``.
+
+ ``dayOfWeek`` accepts a native ``str`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import date
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> d = dfn.lit(pa.scalar(date(2020, 1, 15), type=pa.date32()))
+ >>> r = df.select(dfn.functions.spark.next_day(d, "Mon").alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ datetime.date(2020, 1, 20)
+
+
+.. py:function:: parse_url(url: datafusion.expr.Expr, partToExtract: datafusion.expr.Expr | str, key: datafusion.expr.Expr | str | None = None) -> datafusion.expr.Expr
+
+ Spark ``parse_url``: extract a part from a URL; errors on invalid URLs.
+
+ ``partToExtract`` is one of ``"HOST"``, ``"PATH"``, ``"QUERY"``,
+ ``"REF"``, ``"PROTOCOL"``, ``"FILE"``, ``"AUTHORITY"``, ``"USERINFO"``.
+ Pass ``key`` only with ``"QUERY"`` to extract a single parameter. Bare
+ ``str`` values for ``partToExtract``/``key`` are treated as column names
+ (matching pyspark); pass :func:`~datafusion.lit` for a literal.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.parse_url(
+ ... dfn.lit("http://example.com/path?q=1"), dfn.lit("HOST")
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ 'example.com'
+
+ >>> r = df.select(
+ ... dfn.functions.spark.parse_url(
+ ... dfn.lit("http://example.com/path?q=1"),
+ ... dfn.lit("QUERY"),
+ ... key=dfn.lit("q"),
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ '1'
+
+
+.. py:function:: pmod(dividend: datafusion.expr.Expr | float, divisor: datafusion.expr.Expr | float) -> datafusion.expr.Expr
+
+ Spark ``pmod``: positive remainder of division.
+
+ ``dividend`` and ``divisor`` accept native numbers or :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.pmod(dfn.lit(-1), dfn.lit(3)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 2
+
+
+.. py:function:: rint(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``rint``: round to nearest mathematical integer (as double).
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.rint(dfn.lit(2.5)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 2.0
+
+
+.. py:function:: round(col: datafusion.expr.Expr, scale: datafusion.expr.Expr | int | None = None) -> datafusion.expr.Expr
+
+ Spark ``round``: round to ``scale`` decimal places, HALF_UP rounding.
+
+ ``scale`` defaults to zero when omitted, matching pyspark, and accepts a
+ native ``int`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.round(dfn.lit(2.5)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 3.0
+
+ >>> r = df.select(
+ ... dfn.functions.spark.round(dfn.lit(2.345), scale=2).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 2.35
+
+
+.. py:function:: sec(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``sec``: secant.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.sec(dfn.lit(0.0)).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 1.0
+
+
+.. py:function:: second(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``second``: extract second component of a timestamp.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import datetime
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> ts = dfn.lit(
+ ... pa.scalar(datetime(2020, 1, 15, 14, 30, 45),
+ ... type=pa.timestamp('us')))
+ >>> r = df.select(dfn.functions.spark.second(ts).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 45
+
+
+.. py:function:: sha1(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``sha1``: SHA-1 hash as a hex string.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"s": ["hello"]})
+ >>> r = df.select(dfn.functions.spark.sha1(dfn.col("s")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 'aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d'
+
+
+.. py:function:: sha2(col: datafusion.expr.Expr, numBits: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+
+ Spark ``sha2``: SHA-2 family hash (224, 256, 384, 512). Bit length 0 = 256.
+
+ ``numBits`` accepts a native ``int`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"s": ["hello"]})
+ >>> r = df.select(
+ ... dfn.functions.spark.sha2(dfn.col("s"), 256).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824'
+
+
+.. py:function:: shiftleft(col: datafusion.expr.Expr, numBits: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+
+ Spark ``shiftleft``: ``value`` shifted left by ``shift`` bits.
+
+ ``numBits`` accepts a native ``int`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.shiftleft(dfn.lit(1), 3).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 8
+
+
+.. py:function:: shiftright(col: datafusion.expr.Expr, numBits: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+
+ Spark ``shiftright``: arithmetic right shift.
+
+ ``numBits`` accepts a native ``int`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.shiftright(dfn.lit(8), 2).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 2
+
+
+.. py:function:: shiftrightunsigned(col: datafusion.expr.Expr, numBits: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+
+ Spark ``shiftrightunsigned``: logical (unsigned) right shift.
+
+ ``numBits`` accepts a native ``int`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.shiftrightunsigned(dfn.lit(8), 2).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 2
+
+
+.. py:function:: shuffle(col: datafusion.expr.Expr, seed: int | None = None) -> datafusion.expr.Expr
+
+ Spark ``shuffle``: returns a random permutation of the input array.
+
+ ``seed`` is accepted for pyspark parity but is not yet wired through the
+ Rust binding; passing a non-``None`` value raises ``NotImplementedError``.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.shuffle(
+ ... dfn.functions.spark.array(dfn.lit(1), dfn.lit(2), dfn.lit(3))
+ ... ).alias("v")
+ ... )
+ >>> sorted(r.collect_column("v")[0].as_py())
+ [1, 2, 3]
+
+
+.. py:function:: size(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``size``: length of an array or map.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.size(
+ ... dfn.functions.spark.array(dfn.lit(1), dfn.lit(2), dfn.lit(3))
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ 3
+
+
+.. py:function:: slice(x: datafusion.expr.Expr, start: datafusion.expr.Expr | int, length: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+
+ Spark ``slice``: subset of the array from 1-indexed ``start`` with ``length``.
+
+ Negative ``start`` counts from the end. ``start`` and ``length`` accept
+ native ``int`` values or :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.slice(
+ ... dfn.functions.spark.array(
+ ... dfn.lit(1), dfn.lit(2), dfn.lit(3), dfn.lit(4)),
+ ... 2, 2,
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ [2, 3]
+
+
+.. py:function:: soundex(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``soundex``: Soundex phonetic code.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.soundex(dfn.lit("Robert")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 'R163'
+
+
+.. py:function:: space(col: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+
+ Spark ``space``: string of n spaces.
+
+ ``col`` accepts a native ``int`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.space(3).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ ' '
+
+
+.. py:function:: spark_cast(arg: datafusion.expr.Expr, type_str: datafusion.expr.Expr | str) -> datafusion.expr.Expr
+
+ Spark ``cast``: cast ``arg`` to the type named by ``type_str``.
+
+ Uses Spark cast semantics (e.g. overflow returns NULL, not error).
+ ``type_str`` accepts a native ``str`` or an :class:`Expr`.
+
+ Currently only supports casting numeric values to ``"timestamp"``.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.spark_cast(
+ ... dfn.lit(1579098645), "timestamp"
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py().isoformat()
+ '2020-01-15T14:30:45+00:00'
+
+
+.. py:function:: str_to_map(text: datafusion.expr.Expr, pairDelim: datafusion.expr.Expr | str | None = None, keyValueDelim: datafusion.expr.Expr | str | None = None) -> datafusion.expr.Expr
+
+ Spark ``str_to_map``: split text into key/value pairs using delimiters.
+
+ Delimiters default to ``","`` and ``":"`` when omitted, matching pyspark.
+ Parameter names match ``pyspark.sql.functions.str_to_map``; pyspark types
+ the delimiters as column-or-name, so a bare ``str`` is treated as a column
+ name. Pass :func:`~datafusion.lit` for a literal delimiter.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.str_to_map(dfn.lit("a:1,b:2")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ [('a', '1'), ('b', '2')]
+
+ >>> r = df.select(
+ ... dfn.functions.spark.str_to_map(
+ ... dfn.lit("a=1;b=2"),
+ ... pairDelim=dfn.lit(";"),
+ ... keyValueDelim=dfn.lit("="),
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ [('a', '1'), ('b', '2')]
+
+
+.. py:function:: substring(str: datafusion.expr.Expr, pos: datafusion.expr.Expr | int, len: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+
+ Spark ``substring``: 1-indexed substring starting at ``pos`` of given ``length``.
+
+ Negative ``pos`` counts from the end. ``pos`` and ``len`` accept native
+ ``int`` values or :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.substring(dfn.lit("hello"), 1, 3).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 'hel'
+
+
+.. py:function:: time_trunc(unit: datafusion.expr.Expr | str, time: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``time_trunc``: truncate time value to unit ``fmt``.
+
+ A bare ``str`` ``unit`` is treated as a column name (matching pyspark),
+ not a literal; pass :func:`~datafusion.lit` for a literal unit.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import time
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> t = dfn.lit(pa.scalar(time(14, 30, 45), type=pa.time64('us')))
+ >>> r = df.select(
+ ... dfn.functions.spark.time_trunc(dfn.lit("hour"), t).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ datetime.time(14, 0)
+
+
+.. py:function:: to_utc_timestamp(timestamp: datafusion.expr.Expr, tz: datafusion.expr.Expr | str) -> datafusion.expr.Expr
+
+ Spark ``to_utc_timestamp``: interpret ``ts`` as ``tz``, convert to UTC.
+
+ ``tz`` accepts a native ``str`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import datetime
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> ts = dfn.lit(
+ ... pa.scalar(datetime(2020, 1, 15, 14, 30, 45),
+ ... type=pa.timestamp('us')))
+ >>> r = df.select(
+ ... dfn.functions.spark.to_utc_timestamp(ts, "UTC").alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ datetime.datetime(2020, 1, 15, 14, 30, 45)
+
+
+.. py:function:: trunc(date: datafusion.expr.Expr, format: datafusion.expr.Expr | str) -> datafusion.expr.Expr
+
+ Spark ``trunc``: truncate date to unit ``fmt``.
+
+ ``format`` accepts a native ``str`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import date
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> d = dfn.lit(pa.scalar(date(2020, 1, 15), type=pa.date32()))
+ >>> r = df.select(dfn.functions.spark.trunc(d, "YEAR").alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ datetime.date(2020, 1, 1)
+
+
+.. py:function:: try_parse_url(url: datafusion.expr.Expr, partToExtract: datafusion.expr.Expr | str, key: datafusion.expr.Expr | str | None = None) -> datafusion.expr.Expr
+
+ Spark ``try_parse_url``: like ``parse_url`` but returns NULL on invalid URLs.
+
+ Bare ``str`` values for ``partToExtract``/``key`` are treated as column
+ names (matching pyspark); pass :func:`~datafusion.lit` for a literal.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.try_parse_url(
+ ... dfn.lit("http://example.com/"), dfn.lit("HOST")
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ 'example.com'
+
+
+.. py:function:: try_sum(col: datafusion.expr.Expr, distinct: bool | None = None, filter: datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None, null_treatment: datafusion.common.NullTreatment | None = None) -> datafusion.expr.Expr
+
+ Spark ``try_sum``: sum that returns NULL on overflow.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> r = df.aggregate(
+ ... [], [dfn.functions.spark.try_sum(dfn.col("a")).alias("v")])
+ >>> r.collect_column("v")[0].as_py()
+ 6
+
+
+.. py:function:: try_url_decode(str: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``try_url_decode``: like ``url_decode``; returns NULL on invalid input.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.try_url_decode(dfn.lit("a%20b")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 'a b'
+
+
+.. py:function:: unbase64(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``unbase64``: decode a base64 string to binary.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.unbase64(dfn.lit("aGk=")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ b'hi'
+
+
+.. py:function:: unhex(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``unhex``: convert hexadecimal string to binary.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.unhex(dfn.lit("FF")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ b'\xff'
+
+
+.. py:function:: unix_date(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``unix_date``: days since 1970-01-01 for ``dt``.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import date
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> d = dfn.lit(pa.scalar(date(2020, 1, 15), type=pa.date32()))
+ >>> r = df.select(dfn.functions.spark.unix_date(d).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 18276
+
+
+.. py:function:: unix_micros(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``unix_micros``: microseconds since epoch for ``ts``.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import datetime
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> ts = dfn.lit(
+ ... pa.scalar(datetime(2020, 1, 15, 14, 30, 45),
+ ... type=pa.timestamp('us')))
+ >>> r = df.select(dfn.functions.spark.unix_micros(ts).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 1579098645000000
+
+
+.. py:function:: unix_millis(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``unix_millis``: milliseconds since epoch for ``ts``.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import datetime
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> ts = dfn.lit(
+ ... pa.scalar(datetime(2020, 1, 15, 14, 30, 45),
+ ... type=pa.timestamp('us')))
+ >>> r = df.select(dfn.functions.spark.unix_millis(ts).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 1579098645000
+
+
+.. py:function:: unix_seconds(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``unix_seconds``: seconds since epoch for ``ts``.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import datetime
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> ts = dfn.lit(
+ ... pa.scalar(datetime(2020, 1, 15, 14, 30, 45),
+ ... type=pa.timestamp('us')))
+ >>> r = df.select(dfn.functions.spark.unix_seconds(ts).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 1579098645
+
+
+.. py:function:: url_decode(str: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``url_decode``: decode an application/x-www-form-urlencoded string.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.url_decode(dfn.lit("a%20b")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 'a b'
+
+
+.. py:function:: url_encode(str: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``url_encode``: encode a string in application/x-www-form-urlencoded.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.url_encode(dfn.lit("a b")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 'a+b'
+
+
+.. py:function:: width_bucket(v: datafusion.expr.Expr, min: datafusion.expr.Expr, max: datafusion.expr.Expr, numBucket: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+
+ Spark ``width_bucket``: bucket number for ``value`` in equi-width histogram.
+
+ ``numBucket`` accepts a native ``int`` or an :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.width_bucket(
+ ... dfn.lit(5.0), dfn.lit(0.0), dfn.lit(10.0), 5
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ 3
+
+
+.. py:function:: xxhash64(*cols: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``xxhash64``: 64-bit xxHash of the arguments.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(
+ ... dfn.functions.spark.xxhash64(dfn.lit("hello")).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ -4367754540140381902
+
+
diff --git a/_sources/user-guide/common-operations/aggregations.md.txt b/_sources/user-guide/common-operations/aggregations.md.txt
index 32b8ce46..9c2d58e3 100644
--- a/_sources/user-guide/common-operations/aggregations.md.txt
+++ b/_sources/user-guide/common-operations/aggregations.md.txt
@@ -412,6 +412,16 @@ The available aggregate functions are:
\- {py:meth}`datafusion.expr.GroupingSet.rollup`
\- {py:meth}`datafusion.expr.GroupingSet.cube`
\- {py:meth}`datafusion.expr.GroupingSet.grouping_sets`
+11. Spark-Compatible Functions
+ : - {py:func}`datafusion.functions.spark.avg`
+ - {py:func}`datafusion.functions.spark.try_sum`
+ - {py:func}`datafusion.functions.spark.collect_list`
+ - {py:func}`datafusion.functions.spark.collect_set`
+
+The functions in the `datafusion.functions.spark` namespace mirror Apache
+Spark semantics, which can differ from the DataFusion built-ins of the same
+name. They live in a separate namespace so you opt in explicitly. See
+{ref}`spark-functions` for the full catalog and the semantic differences.
## User-Defined Aggregate Functions
diff --git a/_sources/user-guide/common-operations/functions.md.txt b/_sources/user-guide/common-operations/functions.md.txt
index 8f4a05f5..50411825 100644
--- a/_sources/user-guide/common-operations/functions.md.txt
+++ b/_sources/user-guide/common-operations/functions.md.txt
@@ -31,6 +31,12 @@ kernelspec:
DataFusion provides a large number of built-in functions for performing complex queries without requiring user-defined functions.
In here we will cover some of the more popular use cases. If you want to view all the functions go to the {py:mod}`Functions <datafusion.functions>` API Reference.
+```{note}
+For Apache Spark-compatible versions of these functions (with Spark
+NULL-propagation, 1-indexed substrings, HALF_UP rounding, etc.), see
+[Spark-Compatible Functions](spark-functions.md).
+```
+
We'll use the pokemon dataset in the following examples.
```{code-cell} ipython3
diff --git a/_sources/user-guide/common-operations/index.md.txt b/_sources/user-guide/common-operations/index.md.txt
index cf1559e8..d7b0cccd 100644
--- a/_sources/user-guide/common-operations/index.md.txt
+++ b/_sources/user-guide/common-operations/index.md.txt
@@ -30,6 +30,7 @@ select-and-filter
expressions
joins
functions
+spark-functions
aggregations
windows
udf-and-udfa
diff --git a/_sources/user-guide/common-operations/spark-functions.md.txt b/_sources/user-guide/common-operations/spark-functions.md.txt
new file mode 100644
index 00000000..dbe478a4
--- /dev/null
+++ b/_sources/user-guide/common-operations/spark-functions.md.txt
@@ -0,0 +1,91 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(spark-functions)=
+
+# Spark-Compatible Functions
+
+DataFusion ships Spark-compatible versions of a wide set of functions
+(string, math, datetime, hash, array, aggregate) through the upstream
+`datafusion-spark` crate. `datafusion-python` exposes these under
+`datafusion.functions.spark` for use from the DataFrame API, and via
+{py:meth}`~datafusion.SessionContext.enable_spark_functions` for use from
+SQL.
+
+## Why a Separate Namespace?
+
+Several Spark functions share names with DataFusion built-ins but differ in
+semantics. The most common divergences:
+
+- `concat` propagates NULL. `concat('a', NULL, 'b')` returns NULL under
+ Spark semantics, whereas the DataFusion default returns `'ab'`.
+- `substring` is 1-indexed and supports negative positions counting from
+ the end of the string.
+- `round` uses HALF_UP rounding mode (`round(2.5, 0) == 3`).
+- Numeric functions (`floor`, `ceil`, `mod`) follow Spark's edge-case
+ handling for negative values and decimals.
+
+Enabling Spark functions does not affect the DataFrame API: you choose which
+implementation to call by which module you import from.
+
+## DataFrame API
+
+Import `spark` and use it like any other functions module. The Spark
+functions can go anywhere you'd put a DataFusion expression — inside
+`select`, `filter`, `with_column`, `aggregate`, and so on.
+
+```python
+from datafusion import SessionContext, col, lit
+from datafusion.functions import spark
+
+ctx = SessionContext()
+df = ctx.from_pydict({"s": ["hello", "world"]})
+
+# SHA-256 hash with Spark semantics
+df.select(spark.sha2(col("s"), lit(256)).alias("h")).show()
+
+# 1-indexed substring
+df.select(spark.substring(col("s"), lit(1), lit(3)).alias("p")).show()
+```
+
+## SQL
+
+To use Spark functions in SQL queries, call
+{py:meth}`~datafusion.SessionContext.enable_spark_functions` on the context.
+This registers every Spark UDF/UDAF/UDWF, overriding any DataFusion built-in
+of the same name.
+
+```python
+from datafusion import SessionContext
+
+ctx = SessionContext()
+ctx.enable_spark_functions()
+
+ctx.sql("SELECT sha2('hello', 256)").show()
+ctx.sql("SELECT concat('a', NULL, 'b')").show() # -> NULL, not 'ab'
+```
+
+The override applies for the lifetime of the session. To call DataFusion's
+built-in versions afterwards, create a fresh `SessionContext`.
+
+## Function Reference
+
+The full, up-to-date list of available Spark functions — with signatures
+and per-function docstrings — lives in the
+{py:mod}`datafusion.functions.spark` API reference.
diff --git a/autoapi/datafusion/catalog/index.html b/autoapi/datafusion/catalog/index.html
index 51cabb47..a27e497c 100644
--- a/autoapi/datafusion/catalog/index.html
+++ b/autoapi/datafusion/catalog/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li>
diff --git a/autoapi/datafusion/context/index.html b/autoapi/datafusion/context/index.html
index ce5b95e0..38f7297a 100644
--- a/autoapi/datafusion/context/index.html
+++ b/autoapi/datafusion/context/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li>
@@ -1246,6 +1250,28 @@ changes here.</p>
</div>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.enable_spark_functions">
+<span class="sig-name descname"><span class="pre">enable_spark_functions</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.context.SessionContext.enable_spark_functions" title="Link to this definition">#</a></dt>
+<dd><p>Register all Spark-compatible functions for SQL access.</p>
+<p>Registers every UDF/UDAF/UDWF from the <code class="docutils literal notranslate"><span class="pre">datafusion-spark</span></code> crate,
+overriding any DataFusion built-ins of the same name with their
+Spark-semantics version (e.g. <code class="docutils literal notranslate"><span class="pre">substring</span></code> becomes 1-indexed,
+<code class="docutils literal notranslate"><span class="pre">concat</span></code> propagates NULL, <code class="docutils literal notranslate"><span class="pre">round</span></code> uses HALF_UP rounding).</p>
+<p>For DataFrame use, import the typed wrappers from
+<a class="reference internal" href="../functions/spark/index.html#module-datafusion.functions.spark" title="datafusion.functions.spark"><code class="xref py py-mod docutils literal notranslate"><span class="pre">datafusion.functions.spark</span></code></a> directly; this method is only
+needed for SQL queries.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">enable_spark_functions</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span>
+<span class="gp">... </span> <span class="s2">"SELECT sha2('hello', 256) AS h"</span>
+<span class="gp">... </span><span class="p">)</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"h"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824'</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.enable_url_table">
<span class="sig-name descname"><span class="pre">enable_url_table</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><span class="pre">SessionContext</span></a></span></span><a class="headerlink" href="#datafusion.context.SessionContext.enable_u [...]
@@ -2523,6 +2549,7 @@ the original session is unchanged.</p>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.context.SessionContext.deregister_udwf"><code class="docutils literal notranslate"><span class="pre">SessionContext.deregister_udwf()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.context.SessionContext.empty_table"><code class="docutils literal notranslate"><span class="pre">SessionContext.empty_table()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.context.SessionContext.enable_ident_normalization"><code class="docutils literal notranslate"><span class="pre">SessionContext.enable_ident_normalization()</span></code></a></li>
+<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.context.SessionContext.enable_spark_functions"><code class="docutils literal notranslate"><span class="pre">SessionContext.enable_spark_functions()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.context.SessionContext.enable_url_table"><code class="docutils literal notranslate"><span class="pre">SessionContext.enable_url_table()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.context.SessionContext.execute"><code class="docutils literal notranslate"><span class="pre">SessionContext.execute()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.context.SessionContext.execute_logical_plan"><code class="docutils literal notranslate"><span class="pre">SessionContext.execute_logical_plan()</span></code></a></li>
diff --git a/autoapi/datafusion/dataframe/index.html b/autoapi/datafusion/dataframe/index.html
index eb942746..5efdb555 100644
--- a/autoapi/datafusion/dataframe/index.html
+++ b/autoapi/datafusion/dataframe/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3 current active"><a class="current reference internal" href="#">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li>
diff --git a/autoapi/datafusion/dataframe_formatter/index.html b/autoapi/datafusion/dataframe_formatter/index.html
index 22704aa6..2cc6e96c 100644
--- a/autoapi/datafusion/dataframe_formatter/index.html
+++ b/autoapi/datafusion/dataframe_formatter/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3 current active"><a class="current reference internal" href="#">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li>
diff --git a/autoapi/datafusion/expr/index.html b/autoapi/datafusion/expr/index.html
index 7e2982e6..2eca9bc8 100644
--- a/autoapi/datafusion/expr/index.html
+++ b/autoapi/datafusion/expr/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3 current active"><a class="current reference internal" href="#">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li>
diff --git a/autoapi/datafusion/functions/index.html b/autoapi/datafusion/functions/index.html
index 9f53962c..8fbe5d98 100644
--- a/autoapi/datafusion/functions/index.html
+++ b/autoapi/datafusion/functions/index.html
@@ -49,7 +49,7 @@
<link rel="icon" href="../../../_static/favicon.svg"/>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
- <link rel="next" title="datafusion.input" href="../input/index.html" />
+ <link rel="next" title="datafusion.functions.spark" href="spark/index.html" />
<link rel="prev" title="datafusion.expr" href="../expr/index.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3 current active"><a class="current reference internal" href="#">datafusion.functions</a></li>
+<li class="toctree-l3 current active has-children"><a class="current reference internal" href="#">datafusion.functions</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li>
@@ -507,6 +511,14 @@ imported as <code class="docutils literal notranslate"><span class="pre">F</span
</div>
<p>See <a class="reference internal" href="../../../user-guide/common-operations/aggregations.html#aggregation"><span class="std std-ref">Aggregation</span></a> and <a class="reference internal" href="../../../user-guide/common-operations/windows.html#window-functions"><span class="std std-ref">Window Functions</span></a> in the online documentation
for categorized catalogs of aggregate and window functions.</p>
+<section id="submodules">
+<h2>Submodules<a class="headerlink" href="#submodules" title="Link to this heading">#</a></h2>
+<div class="toctree-wrapper compound">
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</div>
+</section>
<section id="attributes">
<h2>Attributes<a class="headerlink" href="#attributes" title="Link to this heading">#</a></h2>
<div class="pst-scrollable-table-container"><table class="autosummary longtable table autosummary">
@@ -1480,8 +1492,8 @@ for categorized catalogs of aggregate and window functions.</p>
</table>
</div>
</section>
-<section id="module-contents">
-<h2>Module Contents<a class="headerlink" href="#module-contents" title="Link to this heading">#</a></h2>
+<section id="package-contents">
+<h2>Package Contents<a class="headerlink" href="#package-contents" title="Link to this heading">#</a></h2>
<dl class="py function">
<dt class="sig sig-object py" id="datafusion.functions.abs">
<span class="sig-prename descclassname"><span class="pre">datafusion.functions.</span></span><span class="sig-name descname"><span class="pre">abs</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">arg</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.exp [...]
@@ -7919,11 +7931,11 @@ unchanged. Empty keys raise <code class="xref py py-class docutils literal notra
</div>
</a>
<a class="right-next"
- href="../input/index.html"
+ href="spark/index.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">datafusion.input</p>
+ <p class="prev-next-title">datafusion.functions.spark</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
@@ -7946,9 +7958,10 @@ unchanged. Empty keys raise <code class="xref py py-class docutils literal notra
</div>
<nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2">
<ul class="visible nav section-nav flex-column">
+<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#submodules">Submodules</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#attributes">Attributes</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#functions">Functions</a></li>
-<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#module-contents">Module Contents</a><ul class="visible nav section-nav flex-column">
+<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#package-contents">Package Contents</a><ul class="visible nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.abs"><code class="docutils literal notranslate"><span class="pre">abs()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.acos"><code class="docutils literal notranslate"><span class="pre">acos()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.acosh"><code class="docutils literal notranslate"><span class="pre">acosh()</span></code></a></li>
diff --git a/autoapi/datafusion/functions/spark/index.html b/autoapi/datafusion/functions/spark/index.html
new file mode 100644
index 00000000..6a71814f
--- /dev/null
+++ b/autoapi/datafusion/functions/spark/index.html
@@ -0,0 +1,2458 @@
+
+<!DOCTYPE html>
+
+
+<html lang="en" data-content_root="../../../../" data-theme="auto">
+
+ <head>
+ <meta charset="utf-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
+
+ <title>datafusion.functions.spark — Apache DataFusion in Python documentation</title>
+
+
+
+ <script data-cfasync="false">
+ document.documentElement.dataset.mode = localStorage.getItem("mode") || "auto";
+ document.documentElement.dataset.theme = localStorage.getItem("theme") || "auto";
+ </script>
+ <!--
+ this give us a css class that will be invisible only if js is disabled
+ -->
+ <noscript>
+ <style>
+ .pst-js-only { display: none !important; }
+
+ </style>
+ </noscript>
+
+ <!-- Loaded before other Sphinx assets -->
+ <link href="../../../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
+<link href="../../../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
+
+ <link rel="stylesheet" type="text/css" href="../../../../_static/pygments.css?v=8f2a1f02" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/mystnb.11b39860a7a0cbfd473a3ad8a317855267ff0bd372690045ca344a6b62be495e.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/graphviz.css?v=4ae1632d" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/theme_overrides.css?v=4af573bc" />
+
+ <!-- So that users can add custom icons -->
+ <script src="../../../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
+ <!-- Pre-loaded scripts that we'll load fully later -->
+ <link rel="preload" as="script" href="../../../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
+<link rel="preload" as="script" href="../../../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
+
+ <script src="../../../../_static/documentation_options.js?v=5929fcd5"></script>
+ <script src="../../../../_static/doctools.js?v=9bcbadda"></script>
+ <script src="../../../../_static/sphinx_highlight.js?v=dc90522c"></script>
+ <script>DOCUMENTATION_OPTIONS.pagename = 'autoapi/datafusion/functions/spark/index';</script>
+ <script src="../../../../_static/toc-toggle.js?v=36375f01"></script>
+ <link rel="icon" href="../../../../_static/favicon.svg"/>
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+ <link rel="next" title="datafusion.input" href="../../input/index.html" />
+ <link rel="prev" title="datafusion.functions" href="../index.html" />
+ <meta name="viewport" content="width=device-width, initial-scale=1"/>
+ <meta name="docsearch:language" content="en"/>
+ <meta name="docsearch:version" content="" />
+ </head>
+
+
+ <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="auto">
+
+
+
+ <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
+
+ <div id="pst-scroll-pixel-helper"></div>
+
+ <button type="button" class="btn rounded-pill" id="pst-back-to-top">
+ <i class="fa-solid fa-arrow-up"></i>Back to top</button>
+
+
+ <dialog id="pst-search-dialog">
+
+<form class="bd-search d-flex align-items-center"
+ action="../../../../search.html"
+ method="get">
+ <i class="fa-solid fa-magnifying-glass"></i>
+ <input type="search"
+ class="form-control"
+ name="q"
+ placeholder="Search the docs ..."
+ aria-label="Search the docs ..."
+ autocomplete="off"
+ autocorrect="off"
+ autocapitalize="off"
+ spellcheck="false"/>
+ <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
+</form>
+ </dialog>
+
+ <div class="pst-async-banner-revealer d-none">
+ <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
+</div>
+
+
+ <header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
+<div class="bd-header__inner bd-page-width">
+ <button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
+ <span class="fa-solid fa-bars"></span>
+ </button>
+
+
+ <div class="col-lg-3 navbar-header-items__start">
+
+ <div class="navbar-item">
+
+
+
+
+
+<a class="navbar-brand logo" href="../../../../index.html">
+
+
+
+
+
+
+
+
+
+
+ <img src="../../../../_static/original.svg" class="logo__image only-light" alt="Apache DataFusion in Python"/>
+ <img src="../../../../_static/original_dark.svg" class="logo__image only-dark pst-js-only" alt="Apache DataFusion in Python"/>
+
+
+</a></div>
+
+ </div>
+
+ <div class="col-lg-9 navbar-header-items">
+
+ <div class="me-auto navbar-header-items__center">
+
+ <div class="navbar-item">
+<nav>
+ <ul class="bd-navbar-elements navbar-nav">
+
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../../../user-guide/index.html">
+ User Guide
+ </a>
+</li>
+
+
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../../../contributor-guide/index.html">
+ Contributor Guide
+ </a>
+</li>
+
+
+<li class="nav-item current active">
+ <a class="nav-link nav-internal" href="../../../index.html">
+ API Reference
+ </a>
+</li>
+
+
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../../../links.html">
+ Links
+ </a>
+</li>
+
+ </ul>
+</nav></div>
+
+ </div>
+
+
+ <div class="navbar-header-items__end">
+
+ <div class="navbar-item navbar-persistent--container">
+
+
+<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
+ <i class="fa-solid fa-magnifying-glass"></i>
+ <span class="search-button__default-text">Search</span>
+ <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
+</button>
+ </div>
+
+
+ <div class="navbar-item"><ul class="navbar-icon-links"
+ aria-label="Icon Links">
+ <li class="nav-item">
+
+
+
+
+
+
+
+
+ <a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
+ <span class="sr-only">GitHub</span></a>
+ </li>
+ <li class="nav-item">
+
+
+
+
+
+
+
+
+ <a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i>
+ <span class="sr-only">Rust API docs (docs.rs)</span></a>
+ </li>
+</ul></div>
+
+ <div class="navbar-item">
+
+<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
+ <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
+ <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
+ <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
+</button></div>
+
+ </div>
+
+ </div>
+
+
+ <div class="navbar-persistent--mobile">
+
+<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
+ <i class="fa-solid fa-magnifying-glass"></i>
+ <span class="search-button__default-text">Search</span>
+ <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
+</button>
+ </div>
+
+
+
+ <button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page">
+ <span class="fa-solid fa-outdent"></span>
+ </button>
+
+</div>
+
+ </header>
+
+
+ <div class="bd-container">
+ <div class="bd-container__inner bd-page-width">
+
+
+
+ <dialog id="pst-primary-sidebar-modal"></dialog>
+ <div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
+
+
+
+ <div class="sidebar-header-items sidebar-primary__section">
+
+
+ <div class="sidebar-header-items__center">
+
+
+
+ <div class="navbar-item">
+<nav>
+ <ul class="bd-navbar-elements navbar-nav">
+
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../../../user-guide/index.html">
+ User Guide
+ </a>
+</li>
+
+
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../../../contributor-guide/index.html">
+ Contributor Guide
+ </a>
+</li>
+
+
+<li class="nav-item current active">
+ <a class="nav-link nav-internal" href="../../../index.html">
+ API Reference
+ </a>
+</li>
+
+
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../../../links.html">
+ Links
+ </a>
+</li>
+
+ </ul>
+</nav></div>
+
+
+ </div>
+
+
+
+ <div class="sidebar-header-items__end">
+
+ <div class="navbar-item"><ul class="navbar-icon-links"
+ aria-label="Icon Links">
+ <li class="nav-item">
+
+
+
+
+
+
+
+
+ <a href="https://github.com/apache/datafusion-python" title="GitHub" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-github fa-lg" aria-hidden="true"></i>
+ <span class="sr-only">GitHub</span></a>
+ </li>
+ <li class="nav-item">
+
+
+
+
+
+
+
+
+ <a href="https://docs.rs/datafusion/latest/datafusion/" title="Rust API docs (docs.rs)" class="nav-link pst-navbar-icon" rel="noopener" target="_blank" data-bs-toggle="tooltip" data-bs-placement="bottom"><i class="fa-brands fa-rust fa-lg" aria-hidden="true"></i>
+ <span class="sr-only">Rust API docs (docs.rs)</span></a>
+ </li>
+</ul></div>
+
+ <div class="navbar-item">
+
+<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
+ <i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
+ <i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
+ <i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
+</button></div>
+
+ </div>
+
+ </div>
+
+ <div class="sidebar-primary-items__start sidebar-primary__section">
+ <div class="sidebar-primary-item">
+<nav class="bd-docs-nav bd-links" aria-label="Section Navigation">
+ <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
+ <div class="bd-toc-item navbar-nav">
+ <ul class="current nav bd-sidenav">
+<li class="toctree-l1 has-children"><a class="reference internal" href="../../../../user-guide/index.html">User Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/introduction.html">Introduction</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/basics.html">Concepts</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/data-sources.html">Data Sources</a></li>
+
+<li class="toctree-l2 has-children"><a class="reference internal" href="../../../../user-guide/dataframe/index.html">DataFrames</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/dataframe/rendering.html">DataFrame Rendering</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/dataframe/execution-metrics.html">Execution Metrics</a></li>
+</ul>
+</details></li>
+<li class="toctree-l2 has-children"><a class="reference internal" href="../../../../user-guide/common-operations/index.html">Common Operations</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/views.html">Registering Views</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/basic-info.html">Basic Operations</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/select-and-filter.html">Column Selections</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/expressions.html">Expressions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/joins.html">Joins</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/functions.html">Functions</a></li>
+
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/windows.html">Window Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
+</ul>
+</details></li>
+<li class="toctree-l2 has-children"><a class="reference internal" href="../../../../user-guide/io/index.html">IO</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/io/arrow.html">Arrow</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/io/avro.html">Avro</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/io/csv.html">CSV</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/io/json.html">JSON</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/io/parquet.html">Parquet</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/io/table_provider.html">Custom Table Provider</a></li>
+</ul>
+</details></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/configuration.html">Configuration</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/distributing-work.html">Distributing work</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/sql.html">SQL</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/upgrade-guides.html">Upgrade Guides</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../user-guide/ai-coding-assistants.html">Using AI Coding Assistants</a></li>
+</ul>
+</details></li>
+<li class="toctree-l1 has-children"><a class="reference internal" href="../../../../contributor-guide/index.html">Contributor Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l2"><a class="reference internal" href="../../../../contributor-guide/introduction.html">Introduction</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../contributor-guide/ffi.html">Python Extensions</a></li>
+</ul>
+</details></li>
+<li class="toctree-l1 current active has-children"><a class="reference internal" href="../../../index.html">API Reference</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current">
+<li class="toctree-l2 current active has-children"><a class="reference internal" href="../../index.html">datafusion</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current">
+<li class="toctree-l3"><a class="reference internal" href="../../catalog/index.html">datafusion.catalog</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../context/index.html">datafusion.context</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../dataframe/index.html">datafusion.dataframe</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../expr/index.html">datafusion.expr</a></li>
+<li class="toctree-l3 current active has-children"><a class="reference internal" href="../index.html">datafusion.functions</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current">
+<li class="toctree-l4 current active"><a class="current reference internal" href="#">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../input/base/index.html">datafusion.input.base</a></li>
+<li class="toctree-l4"><a class="reference internal" href="../../input/location/index.html">datafusion.input.location</a></li>
+</ul>
+</details></li>
+<li class="toctree-l3"><a class="reference internal" href="../../io/index.html">datafusion.io</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../ipc/index.html">datafusion.ipc</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../object_store/index.html">datafusion.object_store</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../options/index.html">datafusion.options</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../plan/index.html">datafusion.plan</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../record_batch/index.html">datafusion.record_batch</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../substrait/index.html">datafusion.substrait</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../unparser/index.html">datafusion.unparser</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../user_defined/index.html">datafusion.user_defined</a></li>
+</ul>
+</details></li>
+</ul>
+</details></li>
+<li class="toctree-l1 has-children"><a class="reference internal" href="../../../../links.html">Links</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python">GitHub and Issue Tracker</a></li>
+<li class="toctree-l2"><a class="reference external" href="https://docs.rs/datafusion/latest/datafusion/">Rust API Docs</a></li>
+<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion/blob/main/CODE_OF_CONDUCT.md">Code of Conduct</a></li>
+<li class="toctree-l2"><a class="reference external" href="https://github.com/apache/datafusion-python/tree/main/examples">Examples</a></li>
+</ul>
+</details></li>
+</ul>
+</div>
+</nav></div>
+ </div>
+
+
+ <div class="sidebar-primary-items__end sidebar-primary__section">
+ <div class="sidebar-primary-item">
+<div id="ethical-ad-placement"
+ class="flat"
+ data-ea-publisher="readthedocs"
+ data-ea-type="readthedocs-sidebar"
+ data-ea-manual="true">
+</div></div>
+ </div>
+
+
+ </div>
+
+ <main id="main-content" class="bd-main" role="main">
+
+
+ <div class="bd-content">
+ <div class="bd-article-container">
+
+ <div class="bd-header-article d-print-none">
+<div class="header-article-items header-article__inner">
+
+ <div class="header-article-items__start">
+
+ <div class="header-article-item">
+
+<nav aria-label="Breadcrumb" class="d-print-none">
+ <ul class="bd-breadcrumbs">
+
+ <li class="breadcrumb-item breadcrumb-home">
+ <a href="../../../../index.html" class="nav-link" aria-label="Home">
+ <i class="fa-solid fa-home"></i>
+ </a>
+ </li>
+
+ <li class="breadcrumb-item"><a href="../../../index.html" class="nav-link">API Reference</a></li>
+
+
+ <li class="breadcrumb-item"><a href="../../index.html" class="nav-link">datafusion</a></li>
+
+
+ <li class="breadcrumb-item"><a href="../index.html" class="nav-link">datafusion.functions</a></li>
+
+ <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">datafusion.functions.spark</span></li>
+ </ul>
+</nav>
+</div>
+
+ </div>
+
+
+</div>
+</div>
+
+
+
+
+<div id="searchbox"></div>
+ <article class="bd-article">
+
+ <section id="module-datafusion.functions.spark">
+<span id="datafusion-functions-spark"></span><h1>datafusion.functions.spark<a class="headerlink" href="#module-datafusion.functions.spark" title="Link to this heading">#</a></h1>
+<p>Spark-compatible function bindings.</p>
+<p>These functions mirror the semantics of their Apache Spark counterparts
+exactly. Some override DataFusion built-ins (<code class="docutils literal notranslate"><span class="pre">substring</span></code> is 1-indexed,
+<code class="docutils literal notranslate"><span class="pre">concat</span></code> propagates NULL, <code class="docutils literal notranslate"><span class="pre">round</span></code> uses HALF_UP rounding, etc.), which is
+why they live in a separate namespace rather than replacing the defaults.</p>
+<p>For DataFrame use, import this module and call functions directly. For SQL
+use, call <code class="xref py py-meth docutils literal notranslate"><span class="pre">datafusion.SessionContext.enable_spark_functions()</span></code> to
+register the Spark UDFs by name (overriding any built-ins with matching
+names) before issuing SQL queries.</p>
+<section id="functions">
+<h2>Functions<a class="headerlink" href="#functions" title="Link to this heading">#</a></h2>
+<div class="pst-scrollable-table-container"><table class="autosummary longtable table autosummary">
+<tbody>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.abs" title="datafusion.functions.spark.abs"><code class="xref py py-obj docutils literal notranslate"><span class="pre">abs</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">abs</span></code>: absolute value.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.add_months" title="datafusion.functions.spark.add_months"><code class="xref py py-obj docutils literal notranslate"><span class="pre">add_months</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">add_months</span></code>: date + N months.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.array" title="datafusion.functions.spark.array"><code class="xref py py-obj docutils literal notranslate"><span class="pre">array</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">array</span></code>: builds an array from the given elements.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.array_contains" title="datafusion.functions.spark.array_contains"><code class="xref py py-obj docutils literal notranslate"><span class="pre">array_contains</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">array_contains</span></code>: true if the array contains the element.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.array_repeat" title="datafusion.functions.spark.array_repeat"><code class="xref py py-obj docutils literal notranslate"><span class="pre">array_repeat</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">array_repeat</span></code>: array of <code class="docutils literal notranslate"><span class="pre">element</span></code> repeated <code class="docutils literal notranslate"><span class="pre">count</span></code> times.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.ascii" title="datafusion.functions.spark.ascii"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ascii</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">ascii</span></code>: code point of the first character.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.avg" title="datafusion.functions.spark.avg"><code class="xref py py-obj docutils literal notranslate"><span class="pre">avg</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">avg</span></code>: returns the mean of a numeric column.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.base64" title="datafusion.functions.spark.base64"><code class="xref py py-obj docutils literal notranslate"><span class="pre">base64</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">base64</span></code>: encode binary as a base64 string.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.bin" title="datafusion.functions.spark.bin"><code class="xref py py-obj docutils literal notranslate"><span class="pre">bin</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">bin</span></code>: binary string representation of a long.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.bit_count" title="datafusion.functions.spark.bit_count"><code class="xref py py-obj docutils literal notranslate"><span class="pre">bit_count</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">bit_count</span></code>: number of bits set in the integer's binary form.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.bit_get" title="datafusion.functions.spark.bit_get"><code class="xref py py-obj docutils literal notranslate"><span class="pre">bit_get</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">bit_get</span></code>: returns the bit (0 or 1) at <code class="docutils literal notranslate"><span class="pre">pos</span></code>.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.bitmap_bit_position" title="datafusion.functions.spark.bitmap_bit_position"><code class="xref py py-obj docutils literal notranslate"><span class="pre">bitmap_bit_position</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">bitmap_bit_position</span></code>: bit position for a child expression.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.bitmap_bucket_number" title="datafusion.functions.spark.bitmap_bucket_number"><code class="xref py py-obj docutils literal notranslate"><span class="pre">bitmap_bucket_number</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">bitmap_bucket_number</span></code>: bucket number for a child expression.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.bitmap_count" title="datafusion.functions.spark.bitmap_count"><code class="xref py py-obj docutils literal notranslate"><span class="pre">bitmap_count</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">bitmap_count</span></code>: number of set bits in a bitmap.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.bitwise_not" title="datafusion.functions.spark.bitwise_not"><code class="xref py py-obj docutils literal notranslate"><span class="pre">bitwise_not</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">~</span></code>: bitwise NOT.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.ceil" title="datafusion.functions.spark.ceil"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ceil</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">ceil</span></code>: smallest integer ≥ arg.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.char" title="datafusion.functions.spark.char"><code class="xref py py-obj docutils literal notranslate"><span class="pre">char</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">char</span></code>: ASCII character for a code point (mod 256).</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.collect_list" title="datafusion.functions.spark.collect_list"><code class="xref py py-obj docutils literal notranslate"><span class="pre">collect_list</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">collect_list</span></code>: collect values into an array (preserves dups).</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.collect_set" title="datafusion.functions.spark.collect_set"><code class="xref py py-obj docutils literal notranslate"><span class="pre">collect_set</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">collect_set</span></code>: collect distinct values into an array.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.concat" title="datafusion.functions.spark.concat"><code class="xref py py-obj docutils literal notranslate"><span class="pre">concat</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">concat</span></code>: concatenates strings; NULL if any input is NULL.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.crc32" title="datafusion.functions.spark.crc32"><code class="xref py py-obj docutils literal notranslate"><span class="pre">crc32</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">crc32</span></code>: cyclic redundancy check value as a bigint.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.csc" title="datafusion.functions.spark.csc"><code class="xref py py-obj docutils literal notranslate"><span class="pre">csc</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">csc</span></code>: cosecant.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.date_add" title="datafusion.functions.spark.date_add"><code class="xref py py-obj docutils literal notranslate"><span class="pre">date_add</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">date_add</span></code>: date + N days.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.date_diff" title="datafusion.functions.spark.date_diff"><code class="xref py py-obj docutils literal notranslate"><span class="pre">date_diff</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">date_diff</span></code>: number of days from <code class="docutils literal notranslate"><span class="pre">start_date</span></code> to <code class="docutils literal notranslate"><span class="pre">end_date</span></code>.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.date_part" title="datafusion.functions.spark.date_part"><code class="xref py py-obj docutils literal notranslate"><span class="pre">date_part</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">date_part</span></code>: extract <code class="docutils literal notranslate"><span class="pre">field</span></code> from a date/time/timestamp.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.date_sub" title="datafusion.functions.spark.date_sub"><code class="xref py py-obj docutils literal notranslate"><span class="pre">date_sub</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">date_sub</span></code>: date - N days.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.date_trunc" title="datafusion.functions.spark.date_trunc"><code class="xref py py-obj docutils literal notranslate"><span class="pre">date_trunc</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">date_trunc</span></code>: truncate timestamp to unit <code class="docutils literal notranslate"><span class="pre">fmt</span></code>.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.elt" title="datafusion.functions.spark.elt"><code class="xref py py-obj docutils literal notranslate"><span class="pre">elt</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">elt</span></code>: returns the n-th input (1-indexed).</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.expm1" title="datafusion.functions.spark.expm1"><code class="xref py py-obj docutils literal notranslate"><span class="pre">expm1</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">expm1</span></code>: exp(arg) - 1.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.factorial" title="datafusion.functions.spark.factorial"><code class="xref py py-obj docutils literal notranslate"><span class="pre">factorial</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">factorial</span></code>: n! for n in [0..20], else NULL.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.floor" title="datafusion.functions.spark.floor"><code class="xref py py-obj docutils literal notranslate"><span class="pre">floor</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">floor</span></code>: largest integer ≤ arg.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.format_string" title="datafusion.functions.spark.format_string"><code class="xref py py-obj docutils literal notranslate"><span class="pre">format_string</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">format_string</span></code>: printf-style format string.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.from_utc_timestamp" title="datafusion.functions.spark.from_utc_timestamp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">from_utc_timestamp</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">from_utc_timestamp</span></code>: interpret <code class="docutils literal notranslate"><span class="pre">ts</span></code> as UTC, convert to <code class="docutils literal notranslate"><span class="pre">tz</span></code>.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.hex" title="datafusion.functions.spark.hex"><code class="xref py py-obj docutils literal notranslate"><span class="pre">hex</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">hex</span></code>: hexadecimal representation.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.hour" title="datafusion.functions.spark.hour"><code class="xref py py-obj docutils literal notranslate"><span class="pre">hour</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">hour</span></code>: extract hour component of a timestamp.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.if_" title="datafusion.functions.spark.if_"><code class="xref py py-obj docutils literal notranslate"><span class="pre">if_</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">if</span></code>: returns <code class="docutils literal notranslate"><span class="pre">if_true</span></code> when <code class="docutils literal notranslate"><span class="pre">condition</span></code> is true, else <code class="docutils literal notranslate"><span class="pre">if_false</span></code>.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.ilike" title="datafusion.functions.spark.ilike"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ilike</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">ilike</span></code>: case-insensitive pattern match.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.is_valid_utf8" title="datafusion.functions.spark.is_valid_utf8"><code class="xref py py-obj docutils literal notranslate"><span class="pre">is_valid_utf8</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">is_valid_utf8</span></code>: true if the string is valid UTF-8.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.json_tuple" title="datafusion.functions.spark.json_tuple"><code class="xref py py-obj docutils literal notranslate"><span class="pre">json_tuple</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">json_tuple</span></code>: extract top-level fields from a JSON string.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.last_day" title="datafusion.functions.spark.last_day"><code class="xref py py-obj docutils literal notranslate"><span class="pre">last_day</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">last_day</span></code>: last day of the month containing the date.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.length" title="datafusion.functions.spark.length"><code class="xref py py-obj docutils literal notranslate"><span class="pre">length</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">length</span></code>: character length of a string, or byte length of binary.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.like" title="datafusion.functions.spark.like"><code class="xref py py-obj docutils literal notranslate"><span class="pre">like</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">like</span></code>: case-sensitive pattern match.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.luhn_check" title="datafusion.functions.spark.luhn_check"><code class="xref py py-obj docutils literal notranslate"><span class="pre">luhn_check</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">luhn_check</span></code>: true if the digit string passes the Luhn check.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.make_dt_interval" title="datafusion.functions.spark.make_dt_interval"><code class="xref py py-obj docutils literal notranslate"><span class="pre">make_dt_interval</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">make_dt_interval</span></code>: day-time interval from components.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.make_interval" title="datafusion.functions.spark.make_interval"><code class="xref py py-obj docutils literal notranslate"><span class="pre">make_interval</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">make_interval</span></code>: interval from year/month/week/day/hour/min/sec parts.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.make_valid_utf8" title="datafusion.functions.spark.make_valid_utf8"><code class="xref py py-obj docutils literal notranslate"><span class="pre">make_valid_utf8</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">make_valid_utf8</span></code>: replace invalid UTF-8 bytes with U+FFFD.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.map_from_arrays" title="datafusion.functions.spark.map_from_arrays"><code class="xref py py-obj docutils literal notranslate"><span class="pre">map_from_arrays</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">map_from_arrays</span></code>: build a map from parallel key/value arrays.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.map_from_entries" title="datafusion.functions.spark.map_from_entries"><code class="xref py py-obj docutils literal notranslate"><span class="pre">map_from_entries</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">map_from_entries</span></code>: build a map from an array of key/value structs.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.minute" title="datafusion.functions.spark.minute"><code class="xref py py-obj docutils literal notranslate"><span class="pre">minute</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">minute</span></code>: extract minute component of a timestamp.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.modulus" title="datafusion.functions.spark.modulus"><code class="xref py py-obj docutils literal notranslate"><span class="pre">modulus</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">mod</span></code>: remainder of <code class="docutils literal notranslate"><span class="pre">dividend</span> <span class="pre">/</span> <span class="pre">divisor</span></code> (sign follows dividend).</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.negative" title="datafusion.functions.spark.negative"><code class="xref py py-obj docutils literal notranslate"><span class="pre">negative</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">negative</span></code>: unary minus.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.next_day" title="datafusion.functions.spark.next_day"><code class="xref py py-obj docutils literal notranslate"><span class="pre">next_day</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">next_day</span></code>: first date after <code class="docutils literal notranslate"><span class="pre">start_date</span></code> named <code class="docutils literal notranslate"><span class="pre">day_of_week</span></code>.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.parse_url" title="datafusion.functions.spark.parse_url"><code class="xref py py-obj docutils literal notranslate"><span class="pre">parse_url</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">parse_url</span></code>: extract a part from a URL; errors on invalid URLs.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.pmod" title="datafusion.functions.spark.pmod"><code class="xref py py-obj docutils literal notranslate"><span class="pre">pmod</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">pmod</span></code>: positive remainder of division.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.rint" title="datafusion.functions.spark.rint"><code class="xref py py-obj docutils literal notranslate"><span class="pre">rint</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">rint</span></code>: round to nearest mathematical integer (as double).</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.round" title="datafusion.functions.spark.round"><code class="xref py py-obj docutils literal notranslate"><span class="pre">round</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">round</span></code>: round to <code class="docutils literal notranslate"><span class="pre">scale</span></code> decimal places, HALF_UP rounding.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.sec" title="datafusion.functions.spark.sec"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sec</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">sec</span></code>: secant.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.second" title="datafusion.functions.spark.second"><code class="xref py py-obj docutils literal notranslate"><span class="pre">second</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">second</span></code>: extract second component of a timestamp.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.sha1" title="datafusion.functions.spark.sha1"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sha1</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">sha1</span></code>: SHA-1 hash as a hex string.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.sha2" title="datafusion.functions.spark.sha2"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sha2</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">sha2</span></code>: SHA-2 family hash (224, 256, 384, 512). Bit length 0 = 256.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.shiftleft" title="datafusion.functions.spark.shiftleft"><code class="xref py py-obj docutils literal notranslate"><span class="pre">shiftleft</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">shiftleft</span></code>: <code class="docutils literal notranslate"><span class="pre">value</span></code> shifted left by <code class="docutils literal notranslate"><span class="pre">shift</span></code> bits.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.shiftright" title="datafusion.functions.spark.shiftright"><code class="xref py py-obj docutils literal notranslate"><span class="pre">shiftright</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">shiftright</span></code>: arithmetic right shift.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.shiftrightunsigned" title="datafusion.functions.spark.shiftrightunsigned"><code class="xref py py-obj docutils literal notranslate"><span class="pre">shiftrightunsigned</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">shiftrightunsigned</span></code>: logical (unsigned) right shift.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.shuffle" title="datafusion.functions.spark.shuffle"><code class="xref py py-obj docutils literal notranslate"><span class="pre">shuffle</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">shuffle</span></code>: returns a random permutation of the input array.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.size" title="datafusion.functions.spark.size"><code class="xref py py-obj docutils literal notranslate"><span class="pre">size</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">size</span></code>: length of an array or map.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.slice" title="datafusion.functions.spark.slice"><code class="xref py py-obj docutils literal notranslate"><span class="pre">slice</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">slice</span></code>: subset of the array from 1-indexed <code class="docutils literal notranslate"><span class="pre">start</span></code> with <code class="docutils literal notranslate"><span class="pre">length</span></code>.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.soundex" title="datafusion.functions.spark.soundex"><code class="xref py py-obj docutils literal notranslate"><span class="pre">soundex</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">soundex</span></code>: Soundex phonetic code.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.space" title="datafusion.functions.spark.space"><code class="xref py py-obj docutils literal notranslate"><span class="pre">space</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">space</span></code>: string of n spaces.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.spark_cast" title="datafusion.functions.spark.spark_cast"><code class="xref py py-obj docutils literal notranslate"><span class="pre">spark_cast</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">cast</span></code>: cast <code class="docutils literal notranslate"><span class="pre">arg</span></code> to the type named by <code class="docutils literal notranslate"><span class="pre">type_str</span></code>.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.str_to_map" title="datafusion.functions.spark.str_to_map"><code class="xref py py-obj docutils literal notranslate"><span class="pre">str_to_map</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">str_to_map</span></code>: split text into key/value pairs using delimiters.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.substring" title="datafusion.functions.spark.substring"><code class="xref py py-obj docutils literal notranslate"><span class="pre">substring</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">substring</span></code>: 1-indexed substring starting at <code class="docutils literal notranslate"><span class="pre">pos</span></code> of given <code class="docutils literal notranslate"><span class="pre">length</span></code>.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.time_trunc" title="datafusion.functions.spark.time_trunc"><code class="xref py py-obj docutils literal notranslate"><span class="pre">time_trunc</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">time_trunc</span></code>: truncate time value to unit <code class="docutils literal notranslate"><span class="pre">fmt</span></code>.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.to_utc_timestamp" title="datafusion.functions.spark.to_utc_timestamp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">to_utc_timestamp</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">to_utc_timestamp</span></code>: interpret <code class="docutils literal notranslate"><span class="pre">ts</span></code> as <code class="docutils literal notranslate"><span class="pre">tz</span></code>, convert to UTC.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.trunc" title="datafusion.functions.spark.trunc"><code class="xref py py-obj docutils literal notranslate"><span class="pre">trunc</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">trunc</span></code>: truncate date to unit <code class="docutils literal notranslate"><span class="pre">fmt</span></code>.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.try_parse_url" title="datafusion.functions.spark.try_parse_url"><code class="xref py py-obj docutils literal notranslate"><span class="pre">try_parse_url</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">try_parse_url</span></code>: like <code class="docutils literal notranslate"><span class="pre">parse_url</span></code> but returns NULL on invalid URLs.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.try_sum" title="datafusion.functions.spark.try_sum"><code class="xref py py-obj docutils literal notranslate"><span class="pre">try_sum</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">try_sum</span></code>: sum that returns NULL on overflow.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.try_url_decode" title="datafusion.functions.spark.try_url_decode"><code class="xref py py-obj docutils literal notranslate"><span class="pre">try_url_decode</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">try_url_decode</span></code>: like <code class="docutils literal notranslate"><span class="pre">url_decode</span></code>; returns NULL on invalid input.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.unbase64" title="datafusion.functions.spark.unbase64"><code class="xref py py-obj docutils literal notranslate"><span class="pre">unbase64</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">unbase64</span></code>: decode a base64 string to binary.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.unhex" title="datafusion.functions.spark.unhex"><code class="xref py py-obj docutils literal notranslate"><span class="pre">unhex</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">unhex</span></code>: convert hexadecimal string to binary.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.unix_date" title="datafusion.functions.spark.unix_date"><code class="xref py py-obj docutils literal notranslate"><span class="pre">unix_date</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_date</span></code>: days since 1970-01-01 for <code class="docutils literal notranslate"><span class="pre">dt</span></code>.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.unix_micros" title="datafusion.functions.spark.unix_micros"><code class="xref py py-obj docutils literal notranslate"><span class="pre">unix_micros</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_micros</span></code>: microseconds since epoch for <code class="docutils literal notranslate"><span class="pre">ts</span></code>.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.unix_millis" title="datafusion.functions.spark.unix_millis"><code class="xref py py-obj docutils literal notranslate"><span class="pre">unix_millis</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_millis</span></code>: milliseconds since epoch for <code class="docutils literal notranslate"><span class="pre">ts</span></code>.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.unix_seconds" title="datafusion.functions.spark.unix_seconds"><code class="xref py py-obj docutils literal notranslate"><span class="pre">unix_seconds</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_seconds</span></code>: seconds since epoch for <code class="docutils literal notranslate"><span class="pre">ts</span></code>.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.url_decode" title="datafusion.functions.spark.url_decode"><code class="xref py py-obj docutils literal notranslate"><span class="pre">url_decode</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">url_decode</span></code>: decode an application/x-www-form-urlencoded string.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.url_encode" title="datafusion.functions.spark.url_encode"><code class="xref py py-obj docutils literal notranslate"><span class="pre">url_encode</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">url_encode</span></code>: encode a string in application/x-www-form-urlencoded.</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.functions.spark.width_bucket" title="datafusion.functions.spark.width_bucket"><code class="xref py py-obj docutils literal notranslate"><span class="pre">width_bucket</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">width_bucket</span></code>: bucket number for <code class="docutils literal notranslate"><span class="pre">value</span></code> in equi-width histogram.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.functions.spark.xxhash64" title="datafusion.functions.spark.xxhash64"><code class="xref py py-obj docutils literal notranslate"><span class="pre">xxhash64</span></code></a>(→ datafusion.expr.Expr)</p></td>
+<td><p>Spark <code class="docutils literal notranslate"><span class="pre">xxhash64</span></code>: 64-bit xxHash of the arguments.</p></td>
+</tr>
+</tbody>
+</table>
+</div>
+</section>
+<section id="module-contents">
+<h2>Module Contents<a class="headerlink" href="#module-contents" title="Link to this heading">#</a></h2>
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.abs">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">abs</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">dataf [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">abs</span></code>: absolute value.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</ [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">5</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.add_months">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">add_months</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">start</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="p [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">add_months</span></code>: date + N months.</p>
+<p><code class="docutils literal notranslate"><span class="pre">months</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</spa [...]
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">add_months</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="mi">2</span><span class="p [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">datetime.date(2020, 3, 15)</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.array">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">array</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">array</span></code>: builds an array from the given elements.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">[1, 2, 3]</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.array_contains">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">array_contains</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class= [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">array_contains</span></code>: true if the array contains the element.</p>
+<p><code class="docutils literal notranslate"><span class="pre">value</span></code> accepts a native Python literal or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array_contains</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</ [...]
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">True</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.array_repeat">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">array_repeat</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="p [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">array_repeat</span></code>: array of <code class="docutils literal notranslate"><span class="pre">element</span></code> repeated <code class="docutils literal notranslate"><span class="pre">count</span></code> times.</p>
+<p><code class="docutils literal notranslate"><span class="pre">count</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array_repeat</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a"</span><span class="p">),</span> <span class="mi">3</span><span class="p">)</span><span class="o">.</span><span c [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">['a', 'a', 'a']</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.ascii">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">ascii</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">dat [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">ascii</span></code>: code point of the first character.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">ascii</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">( [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">65</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.avg">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">avg</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">dataf [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">avg</span></code>: returns the mean of a numeric column.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">2.0</span><span class="p">,</span> <span class="mf">3.0</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">aggregate</span><span class="p">(</span>
+<span class="gp">... </span> <span class="p">[],</span> <span class="p">[</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">avg</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">))</span><span class="o">.</span><span class="n" [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">2.0</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.base64">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">base64</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">da [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">base64</span></code>: encode binary as a base64 string.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">base64</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p"> [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'aGk='</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.bin">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">bin</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">dataf [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">bin</span></code>: binary string representation of a long.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">bin</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</ [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'111'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.bit_count">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">bit_count</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">bit_count</span></code>: number of bits set in the integer’s binary form.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">bit_count</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class=" [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">3</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.bit_get">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">bit_get</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">d [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">bit_get</span></code>: returns the bit (0 or 1) at <code class="docutils literal notranslate"><span class="pre">pos</span></code>.</p>
+<p>A bare <code class="docutils literal notranslate"><span class="pre">str</span></code> <code class="docutils literal notranslate"><span class="pre">pos</span></code> is treated as a column name (matching pyspark),
+not a literal; pass <a class="reference internal" href="../../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">lit()</span></code></a> for a literal position.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">bit_get</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">5</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</sp [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">1</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.bitmap_bit_position">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">bitmap_bit_position</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span c [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">bitmap_bit_position</span></code>: bit position for a child expression.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">bitmap_bit_position</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">15</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span cla [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">14</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.bitmap_bucket_number">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">bitmap_bucket_number</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">bitmap_bucket_number</span></code>: bucket number for a child expression.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">bitmap_bucket_number</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">15</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span cl [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">1</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.bitmap_count">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">bitmap_count</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="p [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">bitmap_count</span></code>: number of set bits in a bitmap.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">bitmap_count</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="sa">b</span><span class="s2">"</span><span class="se">\xff</span><span class="s2">"</span><span class="p">))</span><span [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">8</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.bitwise_not">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">bitwise_not</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pr [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">~</span></code>: bitwise NOT.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">bitwise_not</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">-1</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.ceil">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">ceil</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">data [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">ceil</span></code>: smallest integer ≥ arg.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">ceil</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(< [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">2</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.char">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">char</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">data [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">char</span></code>: ASCII character for a code point (mod 256).</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">char</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(< [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'A'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.collect_list">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">collect_list</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="p [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">collect_list</span></code>: collect values into an array (preserves dups).</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">aggregate</span><span class="p">(</span>
+<span class="gp">... </span> <span class="p">[],</span> <span class="p">[</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">collect_list</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">))</span><span class="o">.</span><span [...]
+<span class="gp">>>> </span><span class="nb">sorted</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">())</span>
+<span class="go">[1, 2, 2]</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.collect_set">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">collect_set</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pr [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">collect_set</span></code>: collect distinct values into an array.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">aggregate</span><span class="p">(</span>
+<span class="gp">... </span> <span class="p">[],</span> <span class="p">[</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">collect_set</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">))</span><span class="o">.</span><span c [...]
+<span class="gp">>>> </span><span class="nb">sorted</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">())</span>
+<span class="go">[1, 2, 3]</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.concat">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">concat</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">concat</span></code>: concatenates strings; NULL if any input is NULL.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">concat</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a"</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span clas [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'ab'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.crc32">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">crc32</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">dat [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">crc32</span></code>: cyclic redundancy check value as a bigint.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"s"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"ABC"</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">crc32</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">( [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">2743272264</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.csc">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">csc</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">dataf [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">csc</span></code>: cosecant.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">csc</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</ [...]
+<span class="gp">>>> </span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s1">'v'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span><span class="si">:</span><span class="s2">.4f</span><span class="si">}</span><sp [...]
+<span class="go">'1.0000'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.date_add">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">date_add</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">start</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">date_add</span></code>: date + N days.</p>
+<p><code class="docutils literal notranslate"><span class="pre">days</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</spa [...]
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">date_add</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="mi">5</span><span class="p"> [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">datetime.date(2020, 1, 20)</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.date_diff">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">date_diff</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">end</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">date_diff</span></code>: number of days from <code class="docutils literal notranslate"><span class="pre">start_date</span></code> to <code class="docutils literal notranslate"><span class="pre">end_date</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</spa [...]
+<span class="gp">>>> </span><span class="n">end</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">20</span><span class="p">),</s [...]
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">date_diff</span><span class="p">(</span><span class="n">end</span><span class="p">,</span> <span class="n">d</span><span class="p [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">5</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.date_part">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">date_part</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">field</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pr [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">date_part</span></code>: extract <code class="docutils literal notranslate"><span class="pre">field</span></code> from a date/time/timestamp.</p>
+<p><code class="docutils literal notranslate"><span class="pre">field</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</spa [...]
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">date_part</span><span class="p">(</span><span class="s2">"year"</span><span class="p">,</span> <span class="n">d</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">2020</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.date_sub">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">date_sub</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">start</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">date_sub</span></code>: date - N days.</p>
+<p><code class="docutils literal notranslate"><span class="pre">days</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</spa [...]
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">date_sub</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="mi">5</span><span class="p"> [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">datetime.date(2020, 1, 10)</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.date_trunc">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">date_trunc</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">format</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class=" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">date_trunc</span></code>: truncate timestamp to unit <code class="docutils literal notranslate"><span class="pre">fmt</span></code>.</p>
+<p><code class="docutils literal notranslate"><span class="pre">format</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span>
+<span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">date_trunc</span><span class="p">(</span><span class="s2">"month"</span><span class="p">,</span> <span class="n">ts</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">) [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">datetime.datetime(2020, 1, 1, 0, 0)</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.elt">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">elt</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">inputs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">elt</span></code>: returns the n-th input (1-indexed).</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">elt</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a"</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"b"</span><span [...]
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'b'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.expm1">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">expm1</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">dat [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">expm1</span></code>: exp(arg) - 1.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">expm1</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">( [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">0.0</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.factorial">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">factorial</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">factorial</span></code>: n! for n in [0..20], else NULL.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">factorial</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">int32</span><span class="p">()))</span>
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">120</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.floor">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">floor</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">dat [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">floor</span></code>: largest integer ≤ arg.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">floor</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">( [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">1</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.format_string">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">format_string</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">format</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> < [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">format_string</span></code>: printf-style format string.</p>
+<p><code class="docutils literal notranslate"><span class="pre">format</span></code> is the printf-style template (a plain <code class="docutils literal notranslate"><span class="pre">str</span></code> is auto-promoted
+to a literal expression); remaining args are values to substitute.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">format_string</span><span class="p">(</span>
+<span class="gp">... </span> <span class="s2">"</span><span class="si">%d</span><span class="s2">-</span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">42</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"hi"</span><spa [...]
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'42-hi'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.from_utc_timestamp">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">from_utc_timestamp</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">timestamp</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><s [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">from_utc_timestamp</span></code>: interpret <code class="docutils literal notranslate"><span class="pre">ts</span></code> as UTC, convert to <code class="docutils literal notranslate"><span class="pre">tz</span></code>.</p>
+<p><code class="docutils literal notranslate"><span class="pre">tz</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span>
+<span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">from_utc_timestamp</span><span class="p">(</span><span class="n">ts</span><span class="p">,</span> <span class="s2">"UTC"</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">datetime.datetime(2020, 1, 15, 14, 30, 45)</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.hex">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">hex</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">dataf [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">hex</span></code>: hexadecimal representation.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">hex</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</ [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'FF'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.hour">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">hour</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">data [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">hour</span></code>: extract hour component of a timestamp.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span>
+<span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">hour</span><span class="p">(</span><span class="n">ts</span><span class="p">)</span><span class="o">.</span><span class="n">alias [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">14</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.if_">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">if_</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">condition</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">if</span></code>: returns <code class="docutils literal notranslate"><span class="pre">if_true</span></code> when <code class="docutils literal notranslate"><span class="pre">condition</span></code> is true, else <code class="docutils literal notranslate"><span class="pre">if_false</span></code>.</p>
+<p>Exposed as <code class="docutils literal notranslate"><span class="pre">if_</span></code> because <code class="docutils literal notranslate"><span class="pre">if</span></code> is a Python keyword. <code class="docutils literal notranslate"><span class="pre">if_true</span></code> and
+<code class="docutils literal notranslate"><span class="pre">if_false</span></code> accept native Python literals or <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">if_</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="o">></span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="s2">"big"</span><span class="p">,</span> <span class="s2">"small"</span>
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'big'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.ilike">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">ilike</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">dat [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">ilike</span></code>: case-insensitive pattern match.</p>
+<p>A bare <code class="docutils literal notranslate"><span class="pre">str</span></code> <code class="docutils literal notranslate"><span class="pre">pattern</span></code> is treated as a column name (matching pyspark),
+not a literal; pass <a class="reference internal" href="../../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">lit()</span></code></a> for a literal pattern.
+<code class="docutils literal notranslate"><span class="pre">escapeChar</span></code> is accepted for pyspark parity but is not yet wired through
+the Rust binding; passing a non-<code class="docutils literal notranslate"><span class="pre">None</span></code> value raises <code class="docutils literal notranslate"><span class="pre">NotImplementedError</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">ilike</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"HELLO"</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span c [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">True</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.is_valid_utf8">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">is_valid_utf8</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class=" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">is_valid_utf8</span></code>: true if the string is valid UTF-8.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">is_valid_utf8</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"hello"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span> [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">True</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.json_tuple">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">json_tuple</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">json_tuple</span></code>: extract top-level fields from a JSON string.</p>
+<p>Each field name accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">json_tuple</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s1">'{"a":1,"b":"x"}'</span><span class="p">),</span> <span class="s2">"a"</span><span class="p">,</span> <span class="s2">"b"</span>
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">{'c0': '1', 'c1': 'x'}</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.last_day">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">last_day</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre"> [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">last_day</span></code>: last day of the month containing the date.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</spa [...]
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">last_day</span><span class="p">(</span><span class="n">d</span><span class="p">)</span><span class="o">.</span><span class="n">al [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">datetime.date(2020, 1, 31)</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.length">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">length</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">da [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">length</span></code>: character length of a string, or byte length of binary.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">length</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p"> [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">5</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.like">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">like</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">data [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">like</span></code>: case-sensitive pattern match.</p>
+<p>A bare <code class="docutils literal notranslate"><span class="pre">str</span></code> <code class="docutils literal notranslate"><span class="pre">pattern</span></code> is treated as a column name (matching pyspark),
+not a literal; pass <a class="reference internal" href="../../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">lit()</span></code></a> for a literal pattern.
+<code class="docutils literal notranslate"><span class="pre">escapeChar</span></code> is accepted for pyspark parity but is not yet wired through
+the Rust binding; passing a non-<code class="docutils literal notranslate"><span class="pre">None</span></code> value raises <code class="docutils literal notranslate"><span class="pre">NotImplementedError</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">like</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"hello"</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span cl [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">True</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.luhn_check">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">luhn_check</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">luhn_check</span></code>: true if the digit string passes the Luhn check.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">luhn_check</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"4111111111111111"</span><span class="p">)</span>
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">True</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.make_dt_interval">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">make_dt_interval</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">days</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span cla [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">make_dt_interval</span></code>: day-time interval from components.</p>
+<p>All parts are optional; omitted parts default to zero, matching pyspark.
+Integer parts accept a native <code class="docutils literal notranslate"><span class="pre">int</span></code> and <code class="docutils literal notranslate"><span class="pre">secs</span></code> accepts a <code class="docutils literal notranslate"><span class="pre">float</span></code>,
+or any part may be an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">make_dt_interval</span><span class="p">()</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">datetime.timedelta(0)</span>
+</pre></div>
+</div>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">make_dt_interval</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">hours</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">mins</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">secs</span><span class="o">=</span><span class="mf">4.5</span>
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">datetime.timedelta(days=1, seconds=7384, microseconds=500000)</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.make_interval">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">make_interval</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">years</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">make_interval</span></code>: interval from year/month/week/day/hour/min/sec parts.</p>
+<p>All parts are optional; omitted parts default to zero, matching pyspark.
+Integer parts accept a native <code class="docutils literal notranslate"><span class="pre">int</span></code> and <code class="docutils literal notranslate"><span class="pre">secs</span></code> accepts a <code class="docutils literal notranslate"><span class="pre">float</span></code>,
+or any part may be an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">make_interval</span><span class="p">()</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span cl [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span><span class="o">.</span><span class="n">months</span>
+<span class="go">0</span>
+</pre></div>
+</div>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">make_interval</span><span class="p">( [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span><span class="o">.</span><span class="n">months</span>
+<span class="go">12</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.make_valid_utf8">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">make_valid_utf8</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">make_valid_utf8</span></code>: replace invalid UTF-8 bytes with U+FFFD.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">make_valid_utf8</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"hello"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</spa [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'hello'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.map_from_arrays">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">map_from_arrays</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col1</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span clas [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">map_from_arrays</span></code>: build a map from parallel key/value arrays.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">keys</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a"</span><span class="p">),</span> <span class="n">dfn</span><sp [...]
+<span class="gp">>>> </span><span class="n">vals</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">dfn</span><span class="o" [...]
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">map_from_arrays</span><span class="p">(</span><span class="n">keys</span><span class="p">,</span> <span class="n">vals</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">))</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">[('a', 1), ('b', 2)]</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.map_from_entries">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">map_from_entries</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span clas [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">map_from_entries</span></code>: build a map from an array of key/value structs.</p>
+<p><code class="docutils literal notranslate"><span class="pre">col</span></code> must be an array whose elements are two-field structs; the first
+field becomes the map key and the second the value.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">entry_type</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">list_</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">struct</span><span class="p">([(</span><span class="s2">"key"</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">string</span><span class="p">()),</span> <span class="p">(</span><span class="s2">"value"</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">()) [...]
+<span class="gp">>>> </span><span class="n">entries</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">(</span>
+<span class="gp">... </span> <span class="p">[[{</span><span class="s2">"key"</span><span class="p">:</span> <span class="s2">"a"</span><span class="p">,</span> <span class="s2">"value"</span><span class="p">:</span> <span class="mi">1</span><span class="p">},</span> <span class="p">{</span><span class="s2">"key"</span><span class="p">:</span> <span class="s2">"b"</span><span class="p">,</span> <span class="s2">"value"</span> [...]
+<span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">entry_type</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_arrow</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">record_batch</span><span class="p">([</span><span class="n">entries</span><span class="p">],</span> <span class="n">names</span><span class="o">=</span><span class="p">[</span><span class="s2">"e"</span><span class="p">]))</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">map_from_entries</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"e"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span>< [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">[('a', 1), ('b', 2)]</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.minute">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">minute</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">da [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">minute</span></code>: extract minute component of a timestamp.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span>
+<span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">minute</span><span class="p">(</span><span class="n">ts</span><span class="p">)</span><span class="o">.</span><span class="n">ali [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">30</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.modulus">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">modulus</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">dividend</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="p [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">mod</span></code>: remainder of <code class="docutils literal notranslate"><span class="pre">dividend</span> <span class="pre">/</span> <span class="pre">divisor</span></code> (sign follows dividend).</p>
+<p><code class="docutils literal notranslate"><span class="pre">dividend</span></code> and <code class="docutils literal notranslate"><span class="pre">divisor</span></code> accept native numbers or <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">modulus</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">10</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</s [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">1</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.negative">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">negative</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre"> [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">negative</span></code>: unary minus.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">negative</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">-3</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.next_day">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">next_day</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">date</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">next_day</span></code>: first date after <code class="docutils literal notranslate"><span class="pre">start_date</span></code> named <code class="docutils literal notranslate"><span class="pre">day_of_week</span></code>.</p>
+<p><code class="docutils literal notranslate"><span class="pre">dayOfWeek</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</spa [...]
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">next_day</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="s2">"Mon"</span><s [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">datetime.date(2020, 1, 20)</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.parse_url">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">parse_url</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">url</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">parse_url</span></code>: extract a part from a URL; errors on invalid URLs.</p>
+<p><code class="docutils literal notranslate"><span class="pre">partToExtract</span></code> is one of <code class="docutils literal notranslate"><span class="pre">"HOST"</span></code>, <code class="docutils literal notranslate"><span class="pre">"PATH"</span></code>, <code class="docutils literal notranslate"><span class="pre">"QUERY"</span></code>,
+<code class="docutils literal notranslate"><span class="pre">"REF"</span></code>, <code class="docutils literal notranslate"><span class="pre">"PROTOCOL"</span></code>, <code class="docutils literal notranslate"><span class="pre">"FILE"</span></code>, <code class="docutils literal notranslate"><span class="pre">"AUTHORITY"</span></code>, <code class="docutils literal notranslate"><span class="pre">"USERINFO"</span></code>.
+Pass <code class="docutils literal notranslate"><span class="pre">key</span></code> only with <code class="docutils literal notranslate"><span class="pre">"QUERY"</span></code> to extract a single parameter. Bare
+<code class="docutils literal notranslate"><span class="pre">str</span></code> values for <code class="docutils literal notranslate"><span class="pre">partToExtract</span></code>/<code class="docutils literal notranslate"><span class="pre">key</span></code> are treated as column names
+(matching pyspark); pass <a class="reference internal" href="../../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">lit()</span></code></a> for a literal.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">parse_url</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"http://example.com/path?q=1"</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"HOST"</span><span class="p">)</span>
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'example.com'</span>
+</pre></div>
+</div>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">parse_url</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"http://example.com/path?q=1"</span><span class="p">),</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"QUERY"</span><span class="p">),</span>
+<span class="gp">... </span> <span class="n">key</span><span class="o">=</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"q"</span><span class="p">),</span>
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'1'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.pmod">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">pmod</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">dividend</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">pmod</span></code>: positive remainder of division.</p>
+<p><code class="docutils literal notranslate"><span class="pre">dividend</span></code> and <code class="docutils literal notranslate"><span class="pre">divisor</span></code> accept native numbers or <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">pmod</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span> [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">2</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.rint">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">rint</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">data [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">rint</span></code>: round to nearest mathematical integer (as double).</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">rint</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(< [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">2.0</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.round">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">round</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">dat [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">round</span></code>: round to <code class="docutils literal notranslate"><span class="pre">scale</span></code> decimal places, HALF_UP rounding.</p>
+<p><code class="docutils literal notranslate"><span class="pre">scale</span></code> defaults to zero when omitted, matching pyspark, and accepts a
+native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">round</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">( [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">3.0</span>
+</pre></div>
+</div>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">round</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">2.345</span><span class="p">),</span> <span class="n">scale</span><span class="o">=</span><span class="mi">2</span><span class="p">)< [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">2.35</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.sec">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">sec</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">dataf [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">sec</span></code>: secant.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">sec</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</ [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">1.0</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.second">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">second</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">da [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">second</span></code>: extract second component of a timestamp.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span>
+<span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">second</span><span class="p">(</span><span class="n">ts</span><span class="p">)</span><span class="o">.</span><span class="n">ali [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">45</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.sha1">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">sha1</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">data [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">sha1</span></code>: SHA-1 hash as a hex string.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"s"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"hello"</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">sha1</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">(< [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.sha2">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">sha2</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">data [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">sha2</span></code>: SHA-2 family hash (224, 256, 384, 512). Bit length 0 = 256.</p>
+<p><code class="docutils literal notranslate"><span class="pre">numBits</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"s"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"hello"</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">sha2</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"s"</span><span class="p">),</span> <span class="mi">256</span><span class="p">)</span><span class="o">.</span><span class=" [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.shiftleft">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">shiftleft</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">shiftleft</span></code>: <code class="docutils literal notranslate"><span class="pre">value</span></code> shifted left by <code class="docutils literal notranslate"><span class="pre">shift</span></code> bits.</p>
+<p><code class="docutils literal notranslate"><span class="pre">numBits</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">shiftleft</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">alias< [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">8</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.shiftright">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">shiftright</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">shiftright</span></code>: arithmetic right shift.</p>
+<p><code class="docutils literal notranslate"><span class="pre">numBits</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">shiftright</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">8</span><span class="p">),</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">alias [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">2</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.shiftrightunsigned">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">shiftrightunsigned</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span cl [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">shiftrightunsigned</span></code>: logical (unsigned) right shift.</p>
+<p><code class="docutils literal notranslate"><span class="pre">numBits</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">shiftrightunsigned</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">8</span><span class="p">),</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class=" [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">2</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.shuffle">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">shuffle</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">d [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">shuffle</span></code>: returns a random permutation of the input array.</p>
+<p><code class="docutils literal notranslate"><span class="pre">seed</span></code> is accepted for pyspark parity but is not yet wired through the
+Rust binding; passing a non-<code class="docutils literal notranslate"><span class="pre">None</span></code> value raises <code class="docutils literal notranslate"><span class="pre">NotImplementedError</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">shuffle</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</ [...]
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="nb">sorted</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">())</span>
+<span class="go">[1, 2, 3]</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.size">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">size</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">data [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">size</span></code>: length of an array or map.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">size</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</ [...]
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">3</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.slice">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">slice</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">x</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">dataf [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">slice</span></code>: subset of the array from 1-indexed <code class="docutils literal notranslate"><span class="pre">start</span></code> with <code class="docutils literal notranslate"><span class="pre">length</span></code>.</p>
+<p>Negative <code class="docutils literal notranslate"><span class="pre">start</span></code> counts from the end. <code class="docutils literal notranslate"><span class="pre">start</span></code> and <code class="docutils literal notranslate"><span class="pre">length</span></code> accept
+native <code class="docutils literal notranslate"><span class="pre">int</span></code> values or <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">slice</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">array</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">3</span><span class="p">),</span> [...]
+<span class="gp">... </span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span>
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">[2, 3]</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.soundex">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">soundex</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">d [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">soundex</span></code>: Soundex phonetic code.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">soundex</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p" [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'R163'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.space">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">space</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">dat [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">space</span></code>: string of n spaces.</p>
+<p><code class="docutils literal notranslate"><span class="pre">col</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">space</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">alia [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">' '</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.spark_cast">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">spark_cast</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">arg</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">cast</span></code>: cast <code class="docutils literal notranslate"><span class="pre">arg</span></code> to the type named by <code class="docutils literal notranslate"><span class="pre">type_str</span></code>.</p>
+<p>Uses Spark cast semantics (e.g. overflow returns NULL, not error).
+<code class="docutils literal notranslate"><span class="pre">type_str</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p>Currently only supports casting numeric values to <code class="docutils literal notranslate"><span class="pre">"timestamp"</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">spark_cast</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mi">1579098645</span><span class="p">),</span> <span class="s2">"timestamp"</span>
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
+<span class="go">'2020-01-15T14:30:45+00:00'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.str_to_map">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">str_to_map</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">text</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pr [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">str_to_map</span></code>: split text into key/value pairs using delimiters.</p>
+<p>Delimiters default to <code class="docutils literal notranslate"><span class="pre">","</span></code> and <code class="docutils literal notranslate"><span class="pre">":"</span></code> when omitted, matching pyspark.
+Parameter names match <code class="docutils literal notranslate"><span class="pre">pyspark.sql.functions.str_to_map</span></code>; pyspark types
+the delimiters as column-or-name, so a bare <code class="docutils literal notranslate"><span class="pre">str</span></code> is treated as a column
+name. Pass <a class="reference internal" href="../../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">lit()</span></code></a> for a literal delimiter.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">str_to_map</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a:1,b:2"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span>< [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">[('a', '1'), ('b', '2')]</span>
+</pre></div>
+</div>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">str_to_map</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a=1;b=2"</span><span class="p">),</span>
+<span class="gp">... </span> <span class="n">pairDelim</span><span class="o">=</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">";"</span><span class="p">),</span>
+<span class="gp">... </span> <span class="n">keyValueDelim</span><span class="o">=</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"="</span><span class="p">),</span>
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">[('a', '1'), ('b', '2')]</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.substring">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">substring</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">substring</span></code>: 1-indexed substring starting at <code class="docutils literal notranslate"><span class="pre">pos</span></code> of given <code class="docutils literal notranslate"><span class="pre">length</span></code>.</p>
+<p>Negative <code class="docutils literal notranslate"><span class="pre">pos</span></code> counts from the end. <code class="docutils literal notranslate"><span class="pre">pos</span></code> and <code class="docutils literal notranslate"><span class="pre">len</span></code> accept native
+<code class="docutils literal notranslate"><span class="pre">int</span></code> values or <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">substring</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"hello"</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><spa [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'hel'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.time_trunc">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">time_trunc</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">unit</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pr [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">time_trunc</span></code>: truncate time value to unit <code class="docutils literal notranslate"><span class="pre">fmt</span></code>.</p>
+<p>A bare <code class="docutils literal notranslate"><span class="pre">str</span></code> <code class="docutils literal notranslate"><span class="pre">unit</span></code> is treated as a column name (matching pyspark),
+not a literal; pass <a class="reference internal" href="../../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">lit()</span></code></a> for a literal unit.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">time</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">t</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">time</span><span class="p">(</span><span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span [...]
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">time_trunc</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"hour"</span><span class="p">),</span> <span class="n">t</span><span class="p">)</span><span class="o">.</span><span c [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">datetime.time(14, 0)</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.to_utc_timestamp">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">to_utc_timestamp</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">timestamp</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><spa [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">to_utc_timestamp</span></code>: interpret <code class="docutils literal notranslate"><span class="pre">ts</span></code> as <code class="docutils literal notranslate"><span class="pre">tz</span></code>, convert to UTC.</p>
+<p><code class="docutils literal notranslate"><span class="pre">tz</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span>
+<span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">to_utc_timestamp</span><span class="p">(</span><span class="n">ts</span><span class="p">,</span> <span class="s2">"UTC"</span><span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class=" [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">datetime.datetime(2020, 1, 15, 14, 30, 45)</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.trunc">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">trunc</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">date</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">da [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">trunc</span></code>: truncate date to unit <code class="docutils literal notranslate"><span class="pre">fmt</span></code>.</p>
+<p><code class="docutils literal notranslate"><span class="pre">format</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">str</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</spa [...]
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">trunc</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="s2">"YEAR"</span><spa [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">datetime.date(2020, 1, 1)</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.try_parse_url">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">try_parse_url</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">url</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class=" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">try_parse_url</span></code>: like <code class="docutils literal notranslate"><span class="pre">parse_url</span></code> but returns NULL on invalid URLs.</p>
+<p>Bare <code class="docutils literal notranslate"><span class="pre">str</span></code> values for <code class="docutils literal notranslate"><span class="pre">partToExtract</span></code>/<code class="docutils literal notranslate"><span class="pre">key</span></code> are treated as column
+names (matching pyspark); pass <a class="reference internal" href="../../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">lit()</span></code></a> for a literal.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">try_parse_url</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"http://example.com/"</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"HOST"</span><span class="p">)</span>
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'example.com'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.try_sum">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">try_sum</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">d [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">try_sum</span></code>: sum that returns NULL on overflow.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">aggregate</span><span class="p">(</span>
+<span class="gp">... </span> <span class="p">[],</span> <span class="p">[</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">try_sum</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">))</span><span class="o">.</span><span class [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">6</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.try_url_decode">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">try_url_decode</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class= [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">try_url_decode</span></code>: like <code class="docutils literal notranslate"><span class="pre">url_decode</span></code>; returns NULL on invalid input.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">try_url_decode</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a%20b"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'a b'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.unbase64">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">unbase64</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre"> [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">unbase64</span></code>: decode a base64 string to binary.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">unbase64</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"aGk="</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">b'hi'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.unhex">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">unhex</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">dat [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">unhex</span></code>: convert hexadecimal string to binary.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">unhex</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">( [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">b'\xff'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.unix_date">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">unix_date</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre" [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_date</span></code>: days since 1970-01-01 for <code class="docutils literal notranslate"><span class="pre">dt</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">d</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">date</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</spa [...]
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">unix_date</span><span class="p">(</span><span class="n">d</span><span class="p">)</span><span class="o">.</span><span class="n">a [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">18276</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.unix_micros">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">unix_micros</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pr [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_micros</span></code>: microseconds since epoch for <code class="docutils literal notranslate"><span class="pre">ts</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span>
+<span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">unix_micros</span><span class="p">(</span><span class="n">ts</span><span class="p">)</span><span class="o">.</span><span class="n [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">1579098645000000</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.unix_millis">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">unix_millis</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pr [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_millis</span></code>: milliseconds since epoch for <code class="docutils literal notranslate"><span class="pre">ts</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span>
+<span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">unix_millis</span><span class="p">(</span><span class="n">ts</span><span class="p">)</span><span class="o">.</span><span class="n [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">1579098645000</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.unix_seconds">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">unix_seconds</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">col</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="p [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">unix_seconds</span></code>: seconds since epoch for <code class="docutils literal notranslate"><span class="pre">ts</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">ts</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">pa</span><span class="o">.</span><span class="n">scalar</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2020</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">15</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">45</span><span class="p">),</span>
+<span class="gp">... </span> <span class="nb">type</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="s1">'us'</span><span class="p">)))</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">unix_seconds</span><span class="p">(</span><span class="n">ts</span><span class="p">)</span><span class="o">.</span><span class=" [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">1579098645</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.url_decode">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">url_decode</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">url_decode</span></code>: decode an application/x-www-form-urlencoded string.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">url_decode</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a%20b"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><sp [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'a b'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.url_encode">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">url_encode</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">str</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">url_encode</span></code>: encode a string in application/x-www-form-urlencoded.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">url_encode</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"a b"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">'a+b'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.width_bucket">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">width_bucket</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">v</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">width_bucket</span></code>: bucket number for <code class="docutils literal notranslate"><span class="pre">value</span></code> in equi-width histogram.</p>
+<p><code class="docutils literal notranslate"><span class="pre">numBucket</span></code> accepts a native <code class="docutils literal notranslate"><span class="pre">int</span></code> or an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code>.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">width_bucket</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">5.0</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">0.0</span><span class="p">),</span> <span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="mf">10.0</span><span class="p">),</sp [...]
+<span class="gp">... </span> <span class="p">)</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">3</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="datafusion.functions.spark.xxhash64">
+<span class="sig-prename descclassname"><span class="pre">datafusion.functions.spark.</span></span><span class="sig-name descname"><span class="pre">xxhash64</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../../expr/index.html#datafusion.expr.Exp [...]
+<dd><p>Spark <code class="docutils literal notranslate"><span class="pre">xxhash64</span></code>: 64-bit xxHash of the arguments.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">r</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">dfn</span><span class="o">.</span><span class="n">functions</span><span class="o">.</span><span class="n">spark</span><span class="o">.</span><span class="n">xxhash64</span><span class="p">(</span><span class="n">dfn</span><span class="o">.</span><span class="n">lit</span><span class="p">(</span><span class="s2">"hello"</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</span><span [...]
+<span class="gp">>>> </span><span class="n">r</span><span class="o">.</span><span class="n">collect_column</span><span class="p">(</span><span class="s2">"v"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">as_py</span><span class="p">()</span>
+<span class="go">-4367754540140381902</span>
+</pre></div>
+</div>
+</dd></dl>
+
+</section>
+</section>
+
+
+ </article>
+
+
+
+
+
+ <footer class="prev-next-footer d-print-none">
+
+<div class="prev-next-area">
+ <a class="left-prev"
+ href="../index.html"
+ title="previous page">
+ <i class="fa-solid fa-angle-left"></i>
+ <div class="prev-next-info">
+ <p class="prev-next-subtitle">previous</p>
+ <p class="prev-next-title">datafusion.functions</p>
+ </div>
+ </a>
+ <a class="right-next"
+ href="../../input/index.html"
+ title="next page">
+ <div class="prev-next-info">
+ <p class="prev-next-subtitle">next</p>
+ <p class="prev-next-title">datafusion.input</p>
+ </div>
+ <i class="fa-solid fa-angle-right"></i>
+ </a>
+</div>
+ </footer>
+
+ </div>
+
+
+
+ <dialog id="pst-secondary-sidebar-modal"></dialog>
+ <div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
+
+
+ <div class="sidebar-secondary-item">
+<div
+ id="pst-page-navigation-heading-2"
+ class="page-toc tocsection onthispage">
+ <i class="fa-solid fa-list"></i> On this page
+ </div>
+ <nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2">
+ <ul class="visible nav section-nav flex-column">
+<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#functions">Functions</a></li>
+<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#module-contents">Module Contents</a><ul class="visible nav section-nav flex-column">
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.abs"><code class="docutils literal notranslate"><span class="pre">abs()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.add_months"><code class="docutils literal notranslate"><span class="pre">add_months()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.array"><code class="docutils literal notranslate"><span class="pre">array()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.array_contains"><code class="docutils literal notranslate"><span class="pre">array_contains()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.array_repeat"><code class="docutils literal notranslate"><span class="pre">array_repeat()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.ascii"><code class="docutils literal notranslate"><span class="pre">ascii()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.avg"><code class="docutils literal notranslate"><span class="pre">avg()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.base64"><code class="docutils literal notranslate"><span class="pre">base64()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.bin"><code class="docutils literal notranslate"><span class="pre">bin()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.bit_count"><code class="docutils literal notranslate"><span class="pre">bit_count()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.bit_get"><code class="docutils literal notranslate"><span class="pre">bit_get()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.bitmap_bit_position"><code class="docutils literal notranslate"><span class="pre">bitmap_bit_position()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.bitmap_bucket_number"><code class="docutils literal notranslate"><span class="pre">bitmap_bucket_number()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.bitmap_count"><code class="docutils literal notranslate"><span class="pre">bitmap_count()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.bitwise_not"><code class="docutils literal notranslate"><span class="pre">bitwise_not()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.ceil"><code class="docutils literal notranslate"><span class="pre">ceil()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.char"><code class="docutils literal notranslate"><span class="pre">char()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.collect_list"><code class="docutils literal notranslate"><span class="pre">collect_list()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.collect_set"><code class="docutils literal notranslate"><span class="pre">collect_set()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.concat"><code class="docutils literal notranslate"><span class="pre">concat()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.crc32"><code class="docutils literal notranslate"><span class="pre">crc32()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.csc"><code class="docutils literal notranslate"><span class="pre">csc()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.date_add"><code class="docutils literal notranslate"><span class="pre">date_add()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.date_diff"><code class="docutils literal notranslate"><span class="pre">date_diff()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.date_part"><code class="docutils literal notranslate"><span class="pre">date_part()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.date_sub"><code class="docutils literal notranslate"><span class="pre">date_sub()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.date_trunc"><code class="docutils literal notranslate"><span class="pre">date_trunc()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.elt"><code class="docutils literal notranslate"><span class="pre">elt()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.expm1"><code class="docutils literal notranslate"><span class="pre">expm1()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.factorial"><code class="docutils literal notranslate"><span class="pre">factorial()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.floor"><code class="docutils literal notranslate"><span class="pre">floor()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.format_string"><code class="docutils literal notranslate"><span class="pre">format_string()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.from_utc_timestamp"><code class="docutils literal notranslate"><span class="pre">from_utc_timestamp()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.hex"><code class="docutils literal notranslate"><span class="pre">hex()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.hour"><code class="docutils literal notranslate"><span class="pre">hour()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.if_"><code class="docutils literal notranslate"><span class="pre">if_()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.ilike"><code class="docutils literal notranslate"><span class="pre">ilike()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.is_valid_utf8"><code class="docutils literal notranslate"><span class="pre">is_valid_utf8()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.json_tuple"><code class="docutils literal notranslate"><span class="pre">json_tuple()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.last_day"><code class="docutils literal notranslate"><span class="pre">last_day()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.length"><code class="docutils literal notranslate"><span class="pre">length()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.like"><code class="docutils literal notranslate"><span class="pre">like()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.luhn_check"><code class="docutils literal notranslate"><span class="pre">luhn_check()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.make_dt_interval"><code class="docutils literal notranslate"><span class="pre">make_dt_interval()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.make_interval"><code class="docutils literal notranslate"><span class="pre">make_interval()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.make_valid_utf8"><code class="docutils literal notranslate"><span class="pre">make_valid_utf8()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.map_from_arrays"><code class="docutils literal notranslate"><span class="pre">map_from_arrays()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.map_from_entries"><code class="docutils literal notranslate"><span class="pre">map_from_entries()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.minute"><code class="docutils literal notranslate"><span class="pre">minute()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.modulus"><code class="docutils literal notranslate"><span class="pre">modulus()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.negative"><code class="docutils literal notranslate"><span class="pre">negative()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.next_day"><code class="docutils literal notranslate"><span class="pre">next_day()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.parse_url"><code class="docutils literal notranslate"><span class="pre">parse_url()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.pmod"><code class="docutils literal notranslate"><span class="pre">pmod()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.rint"><code class="docutils literal notranslate"><span class="pre">rint()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.round"><code class="docutils literal notranslate"><span class="pre">round()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.sec"><code class="docutils literal notranslate"><span class="pre">sec()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.second"><code class="docutils literal notranslate"><span class="pre">second()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.sha1"><code class="docutils literal notranslate"><span class="pre">sha1()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.sha2"><code class="docutils literal notranslate"><span class="pre">sha2()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.shiftleft"><code class="docutils literal notranslate"><span class="pre">shiftleft()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.shiftright"><code class="docutils literal notranslate"><span class="pre">shiftright()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.shiftrightunsigned"><code class="docutils literal notranslate"><span class="pre">shiftrightunsigned()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.shuffle"><code class="docutils literal notranslate"><span class="pre">shuffle()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.size"><code class="docutils literal notranslate"><span class="pre">size()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.slice"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.soundex"><code class="docutils literal notranslate"><span class="pre">soundex()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.space"><code class="docutils literal notranslate"><span class="pre">space()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.spark_cast"><code class="docutils literal notranslate"><span class="pre">spark_cast()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.str_to_map"><code class="docutils literal notranslate"><span class="pre">str_to_map()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.substring"><code class="docutils literal notranslate"><span class="pre">substring()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.time_trunc"><code class="docutils literal notranslate"><span class="pre">time_trunc()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.to_utc_timestamp"><code class="docutils literal notranslate"><span class="pre">to_utc_timestamp()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.trunc"><code class="docutils literal notranslate"><span class="pre">trunc()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.try_parse_url"><code class="docutils literal notranslate"><span class="pre">try_parse_url()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.try_sum"><code class="docutils literal notranslate"><span class="pre">try_sum()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.try_url_decode"><code class="docutils literal notranslate"><span class="pre">try_url_decode()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.unbase64"><code class="docutils literal notranslate"><span class="pre">unbase64()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.unhex"><code class="docutils literal notranslate"><span class="pre">unhex()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.unix_date"><code class="docutils literal notranslate"><span class="pre">unix_date()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.unix_micros"><code class="docutils literal notranslate"><span class="pre">unix_micros()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.unix_millis"><code class="docutils literal notranslate"><span class="pre">unix_millis()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.unix_seconds"><code class="docutils literal notranslate"><span class="pre">unix_seconds()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.url_decode"><code class="docutils literal notranslate"><span class="pre">url_decode()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.url_encode"><code class="docutils literal notranslate"><span class="pre">url_encode()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.width_bucket"><code class="docutils literal notranslate"><span class="pre">width_bucket()</span></code></a></li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.functions.spark.xxhash64"><code class="docutils literal notranslate"><span class="pre">xxhash64()</span></code></a></li>
+</ul>
+</li>
+</ul>
+ </nav></div>
+
+</div></div>
+
+
+ </div>
+ <footer class="bd-footer-content">
+
+ </footer>
+
+ </main>
+ </div>
+ </div>
+
+ <!-- Scripts loaded after <body> so the DOM is not blocked -->
+ <script defer src="../../../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
+<script defer src="../../../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
+
+<!-- Based on pydata_sphinx_theme/footer.html -->
+<footer class="footer mt-5 mt-md-0">
+ <div class="container">
+
+ <div class="footer-item">
+ <p>Apache Arrow DataFusion, Arrow DataFusion, Apache, the Apache feather logo, and the Apache Arrow DataFusion project logo</p>
+ <p>are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
+ </div>
+ </div>
+</footer>
+
+
+ </body>
+</html>
\ No newline at end of file
diff --git a/autoapi/datafusion/index.html b/autoapi/datafusion/index.html
index 57f95c02..2fb16d7c 100644
--- a/autoapi/datafusion/index.html
+++ b/autoapi/datafusion/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="input/location/index.html">datafusion.input.location</a></li>
diff --git a/autoapi/datafusion/input/base/index.html b/autoapi/datafusion/input/base/index.html
index 11a9c1fd..201e2bdb 100644
--- a/autoapi/datafusion/input/base/index.html
+++ b/autoapi/datafusion/input/base/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 current active has-children"><a class="reference internal" href="../index.html">datafusion.input</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current">
<li class="toctree-l4 current active"><a class="current reference internal" href="#">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../location/index.html">datafusion.input.location</a></li>
diff --git a/autoapi/datafusion/input/index.html b/autoapi/datafusion/input/index.html
index be967aea..4d8b1f32 100644
--- a/autoapi/datafusion/input/index.html
+++ b/autoapi/datafusion/input/index.html
@@ -50,7 +50,7 @@
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
<link rel="next" title="datafusion.input.base" href="base/index.html" />
- <link rel="prev" title="datafusion.functions" href="../functions/index.html" />
+ <link rel="prev" title="datafusion.functions.spark" href="../functions/spark/index.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="" />
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 current active has-children"><a class="current reference internal" href="#">datafusion.input</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="location/index.html">datafusion.input.location</a></li>
@@ -546,12 +550,12 @@
<div class="prev-next-area">
<a class="left-prev"
- href="../functions/index.html"
+ href="../functions/spark/index.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">datafusion.functions</p>
+ <p class="prev-next-title">datafusion.functions.spark</p>
</div>
</a>
<a class="right-next"
diff --git a/autoapi/datafusion/input/location/index.html b/autoapi/datafusion/input/location/index.html
index b57c26c0..4736bfba 100644
--- a/autoapi/datafusion/input/location/index.html
+++ b/autoapi/datafusion/input/location/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 current active has-children"><a class="reference internal" href="../index.html">datafusion.input</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current">
<li class="toctree-l4"><a class="reference internal" href="../base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4 current active"><a class="current reference internal" href="#">datafusion.input.location</a></li>
diff --git a/autoapi/datafusion/io/index.html b/autoapi/datafusion/io/index.html
index e232095c..cd28f428 100644
--- a/autoapi/datafusion/io/index.html
+++ b/autoapi/datafusion/io/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li>
diff --git a/autoapi/datafusion/ipc/index.html b/autoapi/datafusion/ipc/index.html
index 044e9dd9..16759903 100644
--- a/autoapi/datafusion/ipc/index.html
+++ b/autoapi/datafusion/ipc/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li>
diff --git a/autoapi/datafusion/object_store/index.html b/autoapi/datafusion/object_store/index.html
index 1c6b0102..00cc705d 100644
--- a/autoapi/datafusion/object_store/index.html
+++ b/autoapi/datafusion/object_store/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li>
diff --git a/autoapi/datafusion/options/index.html b/autoapi/datafusion/options/index.html
index 6f5669c9..debe9d08 100644
--- a/autoapi/datafusion/options/index.html
+++ b/autoapi/datafusion/options/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li>
diff --git a/autoapi/datafusion/plan/index.html b/autoapi/datafusion/plan/index.html
index 4244995e..5fb7ffb0 100644
--- a/autoapi/datafusion/plan/index.html
+++ b/autoapi/datafusion/plan/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li>
diff --git a/autoapi/datafusion/record_batch/index.html b/autoapi/datafusion/record_batch/index.html
index e789dc4c..c726f00b 100644
--- a/autoapi/datafusion/record_batch/index.html
+++ b/autoapi/datafusion/record_batch/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li>
diff --git a/autoapi/datafusion/substrait/index.html b/autoapi/datafusion/substrait/index.html
index 79396025..73d5e48f 100644
--- a/autoapi/datafusion/substrait/index.html
+++ b/autoapi/datafusion/substrait/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li>
diff --git a/autoapi/datafusion/unparser/index.html b/autoapi/datafusion/unparser/index.html
index 0aea6840..c6163fb0 100644
--- a/autoapi/datafusion/unparser/index.html
+++ b/autoapi/datafusion/unparser/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li>
diff --git a/autoapi/datafusion/user_defined/index.html b/autoapi/datafusion/user_defined/index.html
index a38bf228..8fbc92cd 100644
--- a/autoapi/datafusion/user_defined/index.html
+++ b/autoapi/datafusion/user_defined/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../input/location/index.html">datafusion.input.location</a></li>
diff --git a/autoapi/index.html b/autoapi/index.html
index 19a6ee0f..8fe85548 100644
--- a/autoapi/index.html
+++ b/autoapi/index.html
@@ -360,6 +360,7 @@
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -393,7 +394,10 @@
<li class="toctree-l3"><a class="reference internal" href="datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="datafusion/input/location/index.html">datafusion.input.location</a></li>
@@ -487,7 +491,10 @@
<li class="toctree-l2"><a class="reference internal" href="datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l2"><a class="reference internal" href="datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l2"><a class="reference internal" href="datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l2"><a class="reference internal" href="datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="datafusion/functions/index.html">datafusion.functions</a><ul>
+<li class="toctree-l3"><a class="reference internal" href="datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</li>
<li class="toctree-l2"><a class="reference internal" href="datafusion/input/index.html">datafusion.input</a><ul>
<li class="toctree-l3"><a class="reference internal" href="datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l3"><a class="reference internal" href="datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/contributor-guide/ffi.html b/contributor-guide/ffi.html
index c2c91883..25f74388 100644
--- a/contributor-guide/ffi.html
+++ b/contributor-guide/ffi.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/contributor-guide/index.html b/contributor-guide/index.html
index 5f25c61a..647f16bf 100644
--- a/contributor-guide/index.html
+++ b/contributor-guide/index.html
@@ -360,6 +360,7 @@
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -393,7 +394,10 @@
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/contributor-guide/introduction.html b/contributor-guide/introduction.html
index cbd1a07f..186fbf24 100644
--- a/contributor-guide/introduction.html
+++ b/contributor-guide/introduction.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/genindex.html b/genindex.html
index a88aea01..bd7d2be7 100644
--- a/genindex.html
+++ b/genindex.html
@@ -359,6 +359,7 @@
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -392,7 +393,10 @@
<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
@@ -479,6 +483,7 @@
| <a href="#U"><strong>U</strong></a>
| <a href="#V"><strong>V</strong></a>
| <a href="#W"><strong>W</strong></a>
+ | <a href="#X"><strong>X</strong></a>
| <a href="#Z"><strong>Z</strong></a>
</div>
@@ -985,6 +990,8 @@
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Expr.abs">(datafusion.expr.Expr method)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.abs">(in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.abs">(in module datafusion.functions.spark)</a>
</li>
</ul></li>
<li><a href="autoapi/datafusion/index.html#datafusion.Accumulator">Accumulator (class in datafusion)</a>
@@ -1009,6 +1016,8 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.acosh">(in module datafusion.functions)</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.add_months">add_months() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.add_physical_optimizer_rule">add_physical_optimizer_rule() (datafusion.context.SessionContext method)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Aggregate">Aggregate (in module datafusion.expr)</a>
@@ -1064,7 +1073,11 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.approx_percentile_cont_with_weight">approx_percentile_cont_with_weight() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.array">array() (in module datafusion.functions)</a>
+
+ <ul>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.array">(in module datafusion.functions.spark)</a>
</li>
+ </ul></li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.array_agg">array_agg() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.array_any_match">array_any_match() (in module datafusion.functions)</a>
@@ -1080,7 +1093,11 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.array_concat">array_concat() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.array_contains">array_contains() (in module datafusion.functions)</a>
+
+ <ul>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.array_contains">(in module datafusion.functions.spark)</a>
</li>
+ </ul></li>
<li><a href="autoapi/datafusion/index.html#datafusion.Expr.array_dims">array_dims() (datafusion.Expr method)</a>
<ul>
@@ -1124,11 +1141,11 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.array_indexof">array_indexof() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.array_intersect">array_intersect() (in module datafusion.functions)</a>
-</li>
- <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.array_join">array_join() (in module datafusion.functions)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.array_join">array_join() (in module datafusion.functions)</a>
+</li>
<li><a href="autoapi/datafusion/index.html#datafusion.Expr.array_length">array_length() (datafusion.Expr method)</a>
<ul>
@@ -1184,7 +1201,11 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.array_remove_n">array_remove_n() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.array_repeat">array_repeat() (in module datafusion.functions)</a>
+
+ <ul>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.array_repeat">(in module datafusion.functions.spark)</a>
</li>
+ </ul></li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.array_replace">array_replace() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.array_replace_all">array_replace_all() (in module datafusion.functions)</a>
@@ -1237,6 +1258,8 @@
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Expr.ascii">(datafusion.expr.Expr method)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.ascii">(in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.ascii">(in module datafusion.functions.spark)</a>
</li>
</ul></li>
<li><a href="autoapi/datafusion/index.html#datafusion.Expr.asin">asin() (datafusion.Expr method)</a>
@@ -1274,13 +1297,19 @@
</li>
</ul></li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.avg">avg() (in module datafusion.functions)</a>
+
+ <ul>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.avg">(in module datafusion.functions.spark)</a>
</li>
+ </ul></li>
</ul></td>
</tr></table>
<h2 id="B">B</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.base64">base64() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/input/base/index.html#datafusion.input.base.BaseInputSource">BaseInputSource (class in datafusion.input.base)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Between">Between (in module datafusion.expr)</a>
@@ -1291,9 +1320,15 @@
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Expr.between">(datafusion.expr.Expr method)</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.bin">bin() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.BinaryExpr">BinaryExpr (in module datafusion.expr)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.bit_and">bit_and() (in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.bit_count">bit_count() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.bit_get">bit_get() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/index.html#datafusion.Expr.bit_length">bit_length() (datafusion.Expr method)</a>
@@ -1306,6 +1341,14 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.bit_or">bit_or() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.bit_xor">bit_xor() (in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.bitmap_bit_position">bitmap_bit_position() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.bitmap_bucket_number">bitmap_bucket_number() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.bitmap_count">bitmap_count() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.bitwise_not">bitwise_not() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.ParquetColumnOptions.bloom_filter_enabled">bloom_filter_enabled (datafusion.dataframe.ParquetColumnOptions attribute)</a>
@@ -1313,6 +1356,8 @@
<li><a href="autoapi/datafusion/index.html#datafusion.ParquetColumnOptions.bloom_filter_enabled">(datafusion.ParquetColumnOptions attribute)</a>
</li>
</ul></li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.ParquetColumnOptions.bloom_filter_fpp">bloom_filter_fpp (datafusion.dataframe.ParquetColumnOptions attribute)</a>
<ul>
@@ -1323,8 +1368,6 @@
<li><a href="autoapi/datafusion/index.html#datafusion.ParquetWriterOptions.bloom_filter_fpp">(datafusion.ParquetWriterOptions attribute)</a>
</li>
</ul></li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.ParquetColumnOptions.bloom_filter_ndv">bloom_filter_ndv (datafusion.dataframe.ParquetColumnOptions attribute)</a>
<ul>
@@ -1455,9 +1498,13 @@
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Expr.ceil">(datafusion.expr.Expr method)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.ceil">(in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.ceil">(in module datafusion.functions.spark)</a>
</li>
</ul></li>
<li><a href="autoapi/datafusion/dataframe_formatter/index.html#datafusion.dataframe_formatter.CellFormatter">CellFormatter (class in datafusion.dataframe_formatter)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.char">char() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/index.html#datafusion.Expr.char_length">char_length() (datafusion.Expr method)</a>
@@ -1510,6 +1557,8 @@
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.collect">collect() (datafusion.dataframe.DataFrame method)</a>
</li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.collect_column">collect_column() (datafusion.dataframe.DataFrame method)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.collect_list">collect_list() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/index.html#datafusion.ExecutionPlan.collect_metrics">collect_metrics() (datafusion.ExecutionPlan method)</a>
@@ -1518,6 +1567,8 @@
</li>
</ul></li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.collect_partitioned">collect_partitioned() (datafusion.dataframe.DataFrame method)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.collect_set">collect_set() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/index.html#datafusion.column">column (in module datafusion)</a>
</li>
@@ -1566,7 +1617,11 @@
</li>
</ul></li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.concat">concat() (in module datafusion.functions)</a>
+
+ <ul>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.concat">(in module datafusion.functions.spark)</a>
</li>
+ </ul></li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.concat_ws">concat_ws() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/context/index.html#datafusion.context.RuntimeEnvBuilder.config_internal">config_internal (datafusion.context.RuntimeEnvBuilder attribute)</a>
@@ -1634,6 +1689,8 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.covar_pop">covar_pop() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.covar_samp">covar_samp() (in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.crc32">crc32() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/index.html#datafusion.TableProviderFactory.create">create() (datafusion.TableProviderFactory method)</a>
</li>
@@ -1662,6 +1719,8 @@
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.CreateMemoryTable">CreateMemoryTable (in module datafusion.expr)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.CreateView">CreateView (in module datafusion.expr)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.csc">csc() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/index.html#datafusion.CsvReadOptions">CsvReadOptions (class in datafusion)</a>
@@ -1762,6 +1821,13 @@
<ul>
<li><a href="autoapi/datafusion/functions/index.html#module-datafusion.functions">module</a>
+</li>
+ </ul></li>
+ <li>
+ datafusion.functions.spark
+
+ <ul>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#module-datafusion.functions.spark">module</a>
</li>
</ul></li>
<li>
@@ -1848,18 +1914,34 @@
<li><a href="autoapi/datafusion/user_defined/index.html#module-datafusion.user_defined">module</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.date_add">date_add() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.date_bin">date_bin() (in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.date_diff">date_diff() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.date_format">date_format() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.date_part">date_part() (in module datafusion.functions)</a>
+
+ <ul>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.date_part">(in module datafusion.functions.spark)</a>
+</li>
+ </ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.date_sub">date_sub() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.date_trunc">date_trunc() (in module datafusion.functions)</a>
+
+ <ul>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.date_trunc">(in module datafusion.functions.spark)</a>
</li>
+ </ul></li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.datepart">datepart() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.datetrunc">datetrunc() (in module datafusion.functions)</a>
</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Deallocate">Deallocate (in module datafusion.expr)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.decode">decode() (in module datafusion.functions)</a>
@@ -1868,8 +1950,6 @@
</li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.default_str_repr">default_str_repr() (datafusion.dataframe.DataFrame static method)</a>
</li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/dataframe_formatter/index.html#datafusion.dataframe_formatter.DefaultStyleProvider">DefaultStyleProvider (class in datafusion.dataframe_formatter)</a>
</li>
<li><a href="autoapi/datafusion/index.html#datafusion.Expr.degrees">degrees() (datafusion.Expr method)</a>
@@ -2021,6 +2101,8 @@
</li>
</ul></li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.element_at">element_at() (in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.elt">elt() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/index.html#datafusion.Expr.empty">empty() (datafusion.Expr method)</a>
@@ -2037,6 +2119,8 @@
<li><a href="autoapi/datafusion/dataframe_formatter/index.html#datafusion.dataframe_formatter.DataFrameHtmlFormatter.enable_cell_expansion">enable_cell_expansion (datafusion.dataframe_formatter.DataFrameHtmlFormatter attribute)</a>
</li>
<li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.enable_ident_normalization">enable_ident_normalization() (datafusion.context.SessionContext method)</a>
+</li>
+ <li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.enable_spark_functions">enable_spark_functions() (datafusion.context.SessionContext method)</a>
</li>
<li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.enable_url_table">enable_url_table() (datafusion.context.SessionContext method)</a>
</li>
@@ -2124,6 +2208,8 @@
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.ExplainFormat">(class in datafusion.dataframe)</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.expm1">expm1() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/index.html#datafusion.Expr">Expr (class in datafusion)</a>
<ul>
@@ -2156,6 +2242,8 @@
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Expr.factorial">(datafusion.expr.Expr method)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.factorial">(in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.factorial">(in module datafusion.functions.spark)</a>
</li>
</ul></li>
<li><a href="autoapi/datafusion/index.html#datafusion.CsvReadOptions.file_compression_type">file_compression_type (datafusion.CsvReadOptions attribute)</a>
@@ -2222,6 +2310,8 @@
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Expr.floor">(datafusion.expr.Expr method)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.floor">(in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.floor">(in module datafusion.functions.spark)</a>
</li>
</ul></li>
<li><a href="autoapi/datafusion/dataframe_formatter/index.html#datafusion.dataframe_formatter.DataFrameHtmlFormatter.format_html">format_html() (datafusion.dataframe_formatter.DataFrameHtmlFormatter method)</a>
@@ -2230,6 +2320,8 @@
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.format_string">format_string() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/dataframe_formatter/index.html#datafusion.dataframe_formatter.FormatterManager">FormatterManager (class in datafusion.dataframe_formatter)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.WindowFrameBound.frame_bound">frame_bound (datafusion.expr.WindowFrameBound attribute)</a>
@@ -2302,6 +2394,8 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.from_unixtime">(in module datafusion.functions)</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.from_utc_timestamp">from_utc_timestamp() (in module datafusion.functions.spark)</a>
+</li>
</ul></td>
</tr></table>
@@ -2396,11 +2490,15 @@
<li><a href="autoapi/datafusion/options/index.html#datafusion.options.CsvReadOptions.has_header">(datafusion.options.CsvReadOptions attribute)</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.head">head() (datafusion.dataframe.DataFrame method)</a>
+</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
- <li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.head">head() (datafusion.dataframe.DataFrame method)</a>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.hex">hex() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.HigherOrderFunction">HigherOrderFunction (in module datafusion.expr)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.hour">hour() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/object_store/index.html#datafusion.object_store.Http">Http (in module datafusion.object_store)</a>
</li>
@@ -2410,9 +2508,13 @@
<h2 id="I">I</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.if_">if_() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.ifnull">ifnull() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.ILike">ILike (in module datafusion.expr)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.ilike">ilike() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/user_defined/index.html#datafusion.user_defined.Volatility.Immutable">Immutable (datafusion.user_defined.Volatility attribute)</a>
</li>
@@ -2489,6 +2591,8 @@
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.WindowFrameBound.is_preceding">is_preceding() (datafusion.expr.WindowFrameBound method)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.WindowFrameBound.is_unbounded">is_unbounded() (datafusion.expr.WindowFrameBound method)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.is_valid_utf8">is_valid_utf8() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.IsFalse">IsFalse (in module datafusion.expr)</a>
</li>
@@ -2532,13 +2636,15 @@
</li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.join">join() (datafusion.dataframe.DataFrame method)</a>
</li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.join_on">join_on() (datafusion.dataframe.DataFrame method)</a>
</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.JoinConstraint">JoinConstraint (in module datafusion.expr)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.JoinType">JoinType (in module datafusion.expr)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.json_tuple">json_tuple() (in module datafusion.functions.spark)</a>
</li>
</ul></td>
</tr></table>
@@ -2573,6 +2679,8 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.lambda_var">lambda_var() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.LambdaVariable">LambdaVariable (in module datafusion.expr)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.last_day">last_day() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.last_value">last_value() (in module datafusion.functions)</a>
</li>
@@ -2590,11 +2698,15 @@
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Expr.length">(datafusion.expr.Expr method)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.length">(in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.length">(in module datafusion.functions.spark)</a>
</li>
</ul></li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.levenshtein">levenshtein() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Like">Like (in module datafusion.expr)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.like">like() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Limit">Limit (in module datafusion.expr)</a>
</li>
@@ -2675,11 +2787,11 @@
</li>
</ul></li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.list_normalize">list_normalize() (in module datafusion.functions)</a>
-</li>
- <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.list_overlap">list_overlap() (in module datafusion.functions)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.list_overlap">list_overlap() (in module datafusion.functions)</a>
+</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.list_pop_back">list_pop_back() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.list_pop_front">list_pop_front() (in module datafusion.functions)</a>
@@ -2804,6 +2916,8 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.ltrim">(in module datafusion.functions)</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.luhn_check">luhn_check() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.Compression.LZ4">LZ4 (datafusion.dataframe.Compression attribute)</a>
</li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.Compression.LZ4_RAW">LZ4_RAW (datafusion.dataframe.Compression attribute)</a>
@@ -2817,16 +2931,26 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.make_array">make_array() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.make_date">make_date() (in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.make_dt_interval">make_dt_interval() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.make_interval">make_interval() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.make_list">make_list() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.make_map">make_map() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.make_time">make_time() (in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.make_valid_utf8">make_valid_utf8() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.map_entries">map_entries() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.map_extract">map_extract() (in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.map_from_arrays">map_from_arrays() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.map_from_entries">map_from_entries() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.map_keys">map_keys() (in module datafusion.functions)</a>
</li>
@@ -2886,14 +3010,14 @@
</ul></li>
<li><a href="autoapi/datafusion/catalog/index.html#datafusion.catalog.Schema.memory_schema">memory_schema() (datafusion.catalog.Schema static method)</a>
</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/index.html#datafusion.Accumulator.merge">merge() (datafusion.Accumulator method)</a>
<ul>
<li><a href="autoapi/datafusion/user_defined/index.html#datafusion.user_defined.Accumulator.merge">(datafusion.user_defined.Accumulator method)</a>
</li>
</ul></li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/index.html#datafusion.Metric">Metric (class in datafusion)</a>
<ul>
@@ -2921,6 +3045,8 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.min">min() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/dataframe_formatter/index.html#datafusion.dataframe_formatter.DataFrameHtmlFormatter.min_rows">min_rows (datafusion.dataframe_formatter.DataFrameHtmlFormatter attribute)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.minute">minute() (in module datafusion.functions.spark)</a>
</li>
<li>
module
@@ -2939,6 +3065,8 @@
<li><a href="autoapi/datafusion/expr/index.html#module-datafusion.expr">datafusion.expr</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#module-datafusion.functions">datafusion.functions</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#module-datafusion.functions.spark">datafusion.functions.spark</a>
</li>
<li><a href="autoapi/datafusion/input/index.html#module-datafusion.input">datafusion.input</a>
</li>
@@ -2965,6 +3093,8 @@
<li><a href="autoapi/datafusion/user_defined/index.html#module-datafusion.user_defined">datafusion.user_defined</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.modulus">modulus() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/unparser/index.html#datafusion.unparser.Dialect.mysql">mysql() (datafusion.unparser.Dialect static method)</a>
</li>
</ul></td>
@@ -3006,6 +3136,8 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.nanvl">nanvl() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Negative">Negative (in module datafusion.expr)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.negative">negative() (in module datafusion.functions.spark)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
@@ -3021,6 +3153,8 @@
<li><a href="autoapi/datafusion/index.html#datafusion.RecordBatchStream.next">(datafusion.RecordBatchStream method)</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.next_day">next_day() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Not">Not (in module datafusion.expr)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.now">now() (in module datafusion.functions)</a>
@@ -3133,6 +3267,8 @@
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.parse_sql_expr">(datafusion.dataframe.DataFrame method)</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.parse_url">parse_url() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/index.html#datafusion.Metric.partition">partition (datafusion.Metric property)</a>
<ul>
@@ -3178,6 +3314,8 @@
<li><a href="autoapi/datafusion/substrait/index.html#datafusion.substrait.Plan.plan_internal">plan_internal (datafusion.substrait.Plan attribute)</a>
</li>
<li><a href="autoapi/datafusion/unparser/index.html#datafusion.unparser.Unparser.plan_to_sql">plan_to_sql() (datafusion.unparser.Unparser method)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.pmod">pmod() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.position">position() (in module datafusion.functions)</a>
</li>
@@ -3347,10 +3485,10 @@
</li>
<li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.register_parquet">register_parquet() (datafusion.context.SessionContext method)</a>
</li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.register_record_batches">register_record_batches() (datafusion.context.SessionContext method)</a>
</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/index.html#datafusion.Catalog.register_schema">register_schema() (datafusion.Catalog method)</a>
<ul>
@@ -3448,11 +3586,17 @@
</li>
</ul></li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.right">right() (in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.rint">rint() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.GroupingSet.rollup">rollup() (datafusion.expr.GroupingSet static method)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.round">round() (in module datafusion.functions)</a>
+
+ <ul>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.round">(in module datafusion.functions.spark)</a>
</li>
+ </ul></li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.row">row() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.row_number">row_number() (in module datafusion.functions)</a>
@@ -3534,6 +3678,10 @@
</li>
</ul></li>
<li><a href="autoapi/datafusion/catalog/index.html#datafusion.catalog.SchemaProvider">SchemaProvider (class in datafusion.catalog)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.sec">sec() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.second">second() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.select">select() (datafusion.dataframe.DataFrame method)</a>
</li>
@@ -3580,6 +3728,10 @@
<li><a href="autoapi/datafusion/ipc/index.html#datafusion.ipc.set_worker_ctx">set_worker_ctx() (in module datafusion.ipc)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.SetVariable">SetVariable (in module datafusion.expr)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.sha1">sha1() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.sha2">sha2() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/index.html#datafusion.Expr.sha224">sha224() (datafusion.Expr method)</a>
@@ -3613,9 +3765,17 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.sha512">(in module datafusion.functions)</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.shiftleft">shiftleft() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.shiftright">shiftright() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.shiftrightunsigned">shiftrightunsigned() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.show">show() (datafusion.dataframe.DataFrame method)</a>
</li>
<li><a href="autoapi/datafusion/dataframe_formatter/index.html#datafusion.dataframe_formatter.DataFrameHtmlFormatter.show_truncation_message">show_truncation_message (datafusion.dataframe_formatter.DataFrameHtmlFormatter attribute)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.shuffle">shuffle() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/index.html#datafusion.Expr.signum">signum() (datafusion.Expr method)</a>
@@ -3627,6 +3787,8 @@
</ul></li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.SimilarTo">SimilarTo (in module datafusion.expr)</a>
</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/index.html#datafusion.Expr.sin">sin() (datafusion.Expr method)</a>
<ul>
@@ -3635,8 +3797,6 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.sin">(in module datafusion.functions)</a>
</li>
</ul></li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/index.html#datafusion.Expr.sinh">sinh() (datafusion.Expr method)</a>
<ul>
@@ -3645,12 +3805,16 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.sinh">(in module datafusion.functions)</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.size">size() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.ParquetWriterOptions.skip_arrow_metadata">skip_arrow_metadata (datafusion.dataframe.ParquetWriterOptions attribute)</a>
<ul>
<li><a href="autoapi/datafusion/index.html#datafusion.ParquetWriterOptions.skip_arrow_metadata">(datafusion.ParquetWriterOptions attribute)</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.slice">slice() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.Compression.SNAPPY">SNAPPY (datafusion.dataframe.Compression attribute)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Sort">Sort (in module datafusion.expr)</a>
@@ -3668,6 +3832,12 @@
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.SortExpr">SortExpr (class in datafusion.expr)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.SortKey">SortKey (in module datafusion.expr)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.soundex">soundex() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.space">space() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.spark_cast">spark_cast() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/index.html#datafusion.MetricsSet.spill_count">spill_count (datafusion.MetricsSet property)</a>
@@ -3740,6 +3910,8 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.stddev_pop">stddev_pop() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.stddev_samp">stddev_samp() (in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.str_to_map">str_to_map() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.string_agg">string_agg() (in module datafusion.functions)</a>
</li>
@@ -3770,7 +3942,11 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.substr_index">substr_index() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.substring">substring() (in module datafusion.functions)</a>
+
+ <ul>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.substring">(in module datafusion.functions.spark)</a>
</li>
+ </ul></li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.sum">sum() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/index.html#datafusion.MetricsSet.sum_by_name">sum_by_name() (datafusion.MetricsSet method)</a>
@@ -3861,6 +4037,8 @@
<li><a href="autoapi/datafusion/options/index.html#datafusion.options.CsvReadOptions.terminator">(datafusion.options.CsvReadOptions attribute)</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.time_trunc">time_trunc() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.to_arrow_table">to_arrow_table() (datafusion.dataframe.DataFrame method)</a>
</li>
<li><a href="autoapi/datafusion/index.html#datafusion.ExecutionPlan.to_bytes">to_bytes() (datafusion.ExecutionPlan method)</a>
@@ -3899,12 +4077,12 @@
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.to_local_time">to_local_time() (in module datafusion.functions)</a>
</li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.to_pandas">to_pandas() (datafusion.dataframe.DataFrame method)</a>
</li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.to_polars">to_polars() (datafusion.dataframe.DataFrame method)</a>
</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/index.html#datafusion.ExecutionPlan.to_proto">to_proto() (datafusion.ExecutionPlan method)</a>
<ul>
@@ -3940,6 +4118,8 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.to_timestamp_seconds">to_timestamp_seconds() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.to_unixtime">to_unixtime() (in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.to_utc_timestamp">to_utc_timestamp() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/index.html#datafusion.Expr.to_variant">to_variant() (datafusion.Expr method)</a>
@@ -3982,7 +4162,11 @@
</li>
</ul></li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.trunc">trunc() (in module datafusion.functions)</a>
+
+ <ul>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.trunc">(in module datafusion.functions.spark)</a>
</li>
+ </ul></li>
<li><a href="autoapi/datafusion/index.html#datafusion.CsvReadOptions.truncated_rows">truncated_rows (datafusion.CsvReadOptions attribute)</a>
<ul>
@@ -3996,6 +4180,12 @@
</li>
</ul></li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.try_cast_to_type">try_cast_to_type() (in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.try_parse_url">try_parse_url() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.try_sum">try_sum() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.try_url_decode">try_url_decode() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.TryCast">TryCast (in module datafusion.expr)</a>
</li>
@@ -4069,12 +4259,16 @@
<li><a href="autoapi/datafusion/index.html#datafusion.WindowUDF.udwf">(datafusion.WindowUDF static method)</a>
</li>
</ul></li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.udwfs">udwfs() (datafusion.context.SessionContext method)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.unbase64">unbase64() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.Compression.UNCOMPRESSED">UNCOMPRESSED (datafusion.dataframe.Compression attribute)</a>
</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.unhex">unhex() (in module datafusion.functions.spark)</a>
+</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Union">Union (in module datafusion.expr)</a>
</li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.union">union() (datafusion.dataframe.DataFrame method)</a>
@@ -4086,6 +4280,14 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.union_extract">union_extract() (in module datafusion.functions)</a>
</li>
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.union_tag">union_tag() (in module datafusion.functions)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.unix_date">unix_date() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.unix_micros">unix_micros() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.unix_millis">unix_millis() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.unix_seconds">unix_seconds() (in module datafusion.functions.spark)</a>
</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Unnest">Unnest (in module datafusion.expr)</a>
</li>
@@ -4111,6 +4313,10 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.upper">(in module datafusion.functions)</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.url_decode">url_decode() (in module datafusion.functions.spark)</a>
+</li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.url_encode">url_encode() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/dataframe_formatter/index.html#datafusion.dataframe_formatter.DataFrameHtmlFormatter.use_shared_styles">use_shared_styles (datafusion.dataframe_formatter.DataFrameHtmlFormatter attribute)</a>
</li>
<li><a href="autoapi/datafusion/user_defined/index.html#datafusion.user_defined.WindowEvaluator.uses_window_frame">uses_window_frame() (datafusion.user_defined.WindowEvaluator method)</a>
@@ -4173,6 +4379,8 @@
<li><a href="autoapi/datafusion/functions/index.html#datafusion.functions.when">(in module datafusion.functions)</a>
</li>
</ul></li>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.width_bucket">width_bucket() (in module datafusion.functions.spark)</a>
+</li>
<li><a href="autoapi/datafusion/expr/index.html#datafusion.expr.Window">Window (class in datafusion.expr)</a>
</li>
<li><a href="autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.window">window() (datafusion.dataframe.DataFrame method)</a>
@@ -4480,6 +4688,14 @@
</ul></td>
</tr></table>
+<h2 id="X">X</h2>
+<table style="width: 100%" class="indextable genindextable"><tr>
+ <td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.xxhash64">xxhash64() (in module datafusion.functions.spark)</a>
+</li>
+ </ul></td>
+</tr></table>
+
<h2 id="Z">Z</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
diff --git a/index.html b/index.html
index 01709040..91d7f3cd 100644
--- a/index.html
+++ b/index.html
@@ -365,6 +365,7 @@
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -398,7 +399,10 @@
<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/links.html b/links.html
index 55d58e42..a90f2c82 100644
--- a/links.html
+++ b/links.html
@@ -359,6 +359,7 @@
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -392,7 +393,10 @@
<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/objects.inv b/objects.inv
index 0742681c..47d88368 100644
Binary files a/objects.inv and b/objects.inv differ
diff --git a/py-modindex.html b/py-modindex.html
index 74392dd2..a688d3c9 100644
--- a/py-modindex.html
+++ b/py-modindex.html
@@ -362,6 +362,7 @@
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -395,7 +396,10 @@
<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
@@ -501,6 +505,11 @@
<td>   
<a href="autoapi/datafusion/functions/index.html#module-datafusion.functions"><code class="xref">datafusion.functions</code></a></td><td>
<em></em></td></tr>
+ <tr class="cg-1">
+ <td></td>
+ <td>   
+ <a href="autoapi/datafusion/functions/spark/index.html#module-datafusion.functions.spark"><code class="xref">datafusion.functions.spark</code></a></td><td>
+ <em></em></td></tr>
<tr class="cg-1">
<td></td>
<td>   
diff --git a/search.html b/search.html
index fff58c0c..1617f1b5 100644
--- a/search.html
+++ b/search.html
@@ -361,6 +361,7 @@
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="user-guide/common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -394,7 +395,10 @@
<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/searchindex.js b/searchindex.js
index e5a108ef..de6ccbb4 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles": {"API Reference": [[19, null], [39, "api-reference"]], "Accessing the Calling Session": [[34, "accessing-the-calling-session"]], "Additional Resources": [[41, "additional-resources"]], "Aggregate Functions": [[27, "aggregate-functions"], [34, "aggregate-functions"], [36, "aggregate-functions"]], "Aggregated vs Per-Partition Metrics": [[39, "aggregated-vs-per-partition-metrics"]], "Aggregation": [[27, null]], "Alternative Approach": [[20, "alternative-approac [...]
\ No newline at end of file
+Search.setIndex({"alltitles": {"API Reference": [[20, null], [41, "api-reference"]], "Accessing the Calling Session": [[36, "accessing-the-calling-session"]], "Additional Resources": [[43, "additional-resources"]], "Aggregate Functions": [[28, "aggregate-functions"], [36, "aggregate-functions"], [38, "aggregate-functions"]], "Aggregated vs Per-Partition Metrics": [[41, "aggregated-vs-per-partition-metrics"]], "Aggregation": [[28, null]], "Alternative Approach": [[21, "alternative-approac [...]
\ No newline at end of file
diff --git a/user-guide/ai-coding-assistants.html b/user-guide/ai-coding-assistants.html
index 21599c84..a303e0f1 100644
--- a/user-guide/ai-coding-assistants.html
+++ b/user-guide/ai-coding-assistants.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/basics.html b/user-guide/basics.html
index 0dddaed9..b60084fe 100644
--- a/user-guide/basics.html
+++ b/user-guide/basics.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/common-operations/aggregations.html b/user-guide/common-operations/aggregations.html
index fa71e8f6..0789378a 100644
--- a/user-guide/common-operations/aggregations.html
+++ b/user-guide/common-operations/aggregations.html
@@ -50,7 +50,7 @@
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
<link rel="next" title="Window Functions" href="windows.html" />
- <link rel="prev" title="Functions" href="functions.html" />
+ <link rel="prev" title="Spark-Compatible Functions" href="spark-functions.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="" />
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3 current active"><a class="current reference internal" href="#">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
@@ -682,15 +686,15 @@ Pokemon set. Since there will be many entries of <code class="docutils literal n
+----------+--------------------------------------------------+
| Type 1 | Type 2 List |
+----------+--------------------------------------------------+
-| Water | [Poison, Ice, Psychic, , Flying, Fighting, Dark] |
-| Rock | [Ground, Water, Flying] |
+| Water | [Flying, Psychic, Ice, , Fighting, Poison, Dark] |
+| Rock | [Flying, Ground, Water] |
| Ghost | [Poison] |
-| Ice | [Psychic, Flying] |
-| Dragon | [, Flying] |
-| Grass | [, Psychic, Poison] |
-| Fire | [Dragon, , Flying] |
-| Normal | [Flying, , Fairy] |
-| Poison | [Flying, , Ground] |
+| Ice | [Flying, Psychic] |
+| Dragon | [Flying, ] |
+| Grass | [Poison, Psychic, ] |
+| Fire | [Flying, Dragon, ] |
+| Normal | [Flying, Fairy, ] |
+| Poison | [Ground, Flying, ] |
| Fighting | [] |
+----------+--------------------------------------------------+
Data truncated.
@@ -715,8 +719,8 @@ entries entirely removed. The second is we can use the <code class="docutils lit
+----------+------------------------------------------------+
| Type 1 | Type 2 List |
+----------+------------------------------------------------+
-| Water | [Flying, Dark, Psychic, Ice, Poison, Fighting] |
-| Rock | [Flying, Water, Ground] |
+| Water | [Poison, Ice, Psychic, Dark, Flying, Fighting] |
+| Rock | [Water, Ground, Flying] |
| Ghost | [Poison] |
| Ice | [Psychic, Flying] |
| Dragon | [Flying] |
@@ -1249,7 +1253,21 @@ Where <code class="docutils literal notranslate"><span class="pre">grouping(Type
- <a class="reference internal" href="../../autoapi/datafusion/expr/index.html#datafusion.expr.GroupingSet.rollup" title="datafusion.expr.GroupingSet.rollup"><code class="xref py py-meth docutils literal notranslate"><span class="pre">datafusion.expr.GroupingSet.rollup()</span></code></a>
- <a class="reference internal" href="../../autoapi/datafusion/expr/index.html#datafusion.expr.GroupingSet.cube" title="datafusion.expr.GroupingSet.cube"><code class="xref py py-meth docutils literal notranslate"><span class="pre">datafusion.expr.GroupingSet.cube()</span></code></a>
- <a class="reference internal" href="../../autoapi/datafusion/expr/index.html#datafusion.expr.GroupingSet.grouping_sets" title="datafusion.expr.GroupingSet.grouping_sets"><code class="xref py py-meth docutils literal notranslate"><span class="pre">datafusion.expr.GroupingSet.grouping_sets()</span></code></a></p></li>
+<li><dl class="simple myst">
+<dt>Spark-Compatible Functions</dt><dd><ul class="simple">
+<li><p><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.avg" title="datafusion.functions.spark.avg"><code class="xref py py-func docutils literal notranslate"><span class="pre">datafusion.functions.spark.avg()</span></code></a></p></li>
+<li><p><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.try_sum" title="datafusion.functions.spark.try_sum"><code class="xref py py-func docutils literal notranslate"><span class="pre">datafusion.functions.spark.try_sum()</span></code></a></p></li>
+<li><p><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.collect_list" title="datafusion.functions.spark.collect_list"><code class="xref py py-func docutils literal notranslate"><span class="pre">datafusion.functions.spark.collect_list()</span></code></a></p></li>
+<li><p><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html#datafusion.functions.spark.collect_set" title="datafusion.functions.spark.collect_set"><code class="xref py py-func docutils literal notranslate"><span class="pre">datafusion.functions.spark.collect_set()</span></code></a></p></li>
+</ul>
+</dd>
+</dl>
+</li>
</ol>
+<p>The functions in the <code class="docutils literal notranslate"><span class="pre">datafusion.functions.spark</span></code> namespace mirror Apache
+Spark semantics, which can differ from the DataFusion built-ins of the same
+name. They live in a separate namespace so you opt in explicitly. See
+<a class="reference internal" href="spark-functions.html#spark-functions"><span class="std std-ref">Spark-Compatible Functions</span></a> for the full catalog and the semantic differences.</p>
</section>
<section id="user-defined-aggregate-functions">
<h2>User-Defined Aggregate Functions<a class="headerlink" href="#user-defined-aggregate-functions" title="Link to this heading">#</a></h2>
@@ -1282,12 +1300,12 @@ and must be importable on the receiving worker. See
<div class="prev-next-area">
<a class="left-prev"
- href="functions.html"
+ href="spark-functions.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Functions</p>
+ <p class="prev-next-title">Spark-Compatible Functions</p>
</div>
</a>
<a class="right-next"
diff --git a/user-guide/common-operations/basic-info.html b/user-guide/common-operations/basic-info.html
index f59a5f25..90ee1097 100644
--- a/user-guide/common-operations/basic-info.html
+++ b/user-guide/common-operations/basic-info.html
@@ -360,6 +360,7 @@
<li class="toctree-l3"><a class="reference internal" href="joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="udf-and-udfa.html">User-Defined Functions</a></li>
@@ -393,7 +394,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
@@ -524,11 +528,11 @@
+-----+---------+--------+--------+
| nrs | names | random | groups |
+-----+---------+--------+--------+
-| 1 | python | 657 | A |
-| 2 | ruby | 881 | A |
-| 3 | java | 384 | B |
-| 4 | haskell | 794 | C |
-| 5 | go | 721 | B |
+| 1 | python | 725 | A |
+| 2 | ruby | 85 | A |
+| 3 | java | 363 | B |
+| 4 | haskell | 752 | C |
+| 5 | go | 821 | B |
+-----+---------+--------+--------+
</pre></div>
</div>
@@ -546,8 +550,8 @@
+-----+--------+--------+--------+
| nrs | names | random | groups |
+-----+--------+--------+--------+
-| 1 | python | 657 | A |
-| 2 | ruby | 881 | A |
+| 1 | python | 725 | A |
+| 2 | ruby | 85 | A |
+-----+--------+--------+--------+
</pre></div>
</div>
@@ -579,11 +583,11 @@ passing them to an Arrow table, and then converting them to a pandas DataFrame.<
</div>
<div class="cell_output docutils container">
<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> nrs names random groups
-0 1 python 657 A
-1 2 ruby 881 A
-2 3 java 384 B
-3 4 haskell 794 C
-4 5 go 721 B
+0 1 python 725 A
+1 2 ruby 85 A
+2 3 java 363 B
+3 4 haskell 752 C
+4 5 go 821 B
</pre></div>
</div>
</div>
@@ -602,11 +606,11 @@ passing them to an Arrow table, and then converting them to a pandas DataFrame.<
+------------+--------------------+-------+-------------------+--------+
| count | 5.0 | 5 | 5.0 | 5 |
| null_count | 0.0 | 0 | 0.0 | 0 |
-| mean | 3.0 | null | 687.4 | null |
-| std | 1.5811388300841898 | null | 189.0431167749834 | null |
-| min | 1.0 | go | 384.0 | A |
-| max | 5.0 | ruby | 881.0 | C |
-| median | 3.0 | null | 721.0 | null |
+| mean | 3.0 | null | 549.2 | null |
+| std | 1.5811388300841898 | null | 314.6668079095728 | null |
+| min | 1.0 | go | 85.0 | A |
+| max | 5.0 | ruby | 821.0 | C |
+| median | 3.0 | null | 725.0 | null |
+------------+--------------------+-------+-------------------+--------+
</pre></div>
</div>
diff --git a/user-guide/common-operations/expressions.html b/user-guide/common-operations/expressions.html
index fefd3fe6..96c89fc5 100644
--- a/user-guide/common-operations/expressions.html
+++ b/user-guide/common-operations/expressions.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/common-operations/functions.html b/user-guide/common-operations/functions.html
index cbe91a53..626daca2 100644
--- a/user-guide/common-operations/functions.html
+++ b/user-guide/common-operations/functions.html
@@ -49,7 +49,7 @@
<link rel="icon" href="../../_static/favicon.svg"/>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="Aggregation" href="aggregations.html" />
+ <link rel="next" title="Spark-Compatible Functions" href="spark-functions.html" />
<link rel="prev" title="Joins" href="joins.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="joins.html">Joins</a></li>
<li class="toctree-l3 current active"><a class="current reference internal" href="#">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
@@ -508,6 +512,12 @@
<h1>Functions<a class="headerlink" href="#functions" title="Link to this heading">#</a></h1>
<p>DataFusion provides a large number of built-in functions for performing complex queries without requiring user-defined functions.
In here we will cover some of the more popular use cases. If you want to view all the functions go to the <a class="reference internal" href="../../autoapi/datafusion/functions/index.html#module-datafusion.functions" title="datafusion.functions"><code class="xref py py-mod docutils literal notranslate"><span class="pre">Functions</span></code></a> API Reference.</p>
+<div class="admonition note">
+<p class="admonition-title">Note</p>
+<p>For Apache Spark-compatible versions of these functions (with Spark
+NULL-propagation, 1-indexed substrings, HALF_UP rounding, etc.), see
+<a class="reference internal" href="spark-functions.html"><span class="std std-doc">Spark-Compatible Functions</span></a>.</p>
+</div>
<p>We’ll use the pokemon dataset in the following examples.</p>
<div class="cell docutils container">
<div class="cell_input docutils container">
@@ -601,16 +611,16 @@ In here we will cover some of the more popular use cases. If you want to view al
+-------------------------------+
| now() |
+-------------------------------+
-| 2026-06-16T22:41:50.854565181 |
-| 2026-06-16T22:41:50.854565181 |
-| 2026-06-16T22:41:50.854565181 |
-| 2026-06-16T22:41:50.854565181 |
-| 2026-06-16T22:41:50.854565181 |
-| 2026-06-16T22:41:50.854565181 |
-| 2026-06-16T22:41:50.854565181 |
-| 2026-06-16T22:41:50.854565181 |
-| 2026-06-16T22:41:50.854565181 |
-| 2026-06-16T22:41:50.854565181 |
+| 2026-06-17T18:34:16.686739355 |
+| 2026-06-17T18:34:16.686739355 |
+| 2026-06-17T18:34:16.686739355 |
+| 2026-06-17T18:34:16.686739355 |
+| 2026-06-17T18:34:16.686739355 |
+| 2026-06-17T18:34:16.686739355 |
+| 2026-06-17T18:34:16.686739355 |
+| 2026-06-17T18:34:16.686739355 |
+| 2026-06-17T18:34:16.686739355 |
+| 2026-06-17T18:34:16.686739355 |
+-------------------------------+
Data truncated.
</pre></div>
@@ -857,11 +867,11 @@ Data truncated.
</div>
</a>
<a class="right-next"
- href="aggregations.html"
+ href="spark-functions.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">Aggregation</p>
+ <p class="prev-next-title">Spark-Compatible Functions</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
diff --git a/user-guide/common-operations/index.html b/user-guide/common-operations/index.html
index a9e567b4..b00614c7 100644
--- a/user-guide/common-operations/index.html
+++ b/user-guide/common-operations/index.html
@@ -360,6 +360,7 @@
<li class="toctree-l3"><a class="reference internal" href="joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="udf-and-udfa.html">User-Defined Functions</a></li>
@@ -393,7 +394,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
@@ -540,6 +544,13 @@
<li class="toctree-l2"><a class="reference internal" href="functions.html#fill-null">fill_null</a></li>
</ul>
</li>
+<li class="toctree-l1"><a class="reference internal" href="spark-functions.html">Spark-Compatible Functions</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="spark-functions.html#why-a-separate-namespace">Why a Separate Namespace?</a></li>
+<li class="toctree-l2"><a class="reference internal" href="spark-functions.html#dataframe-api">DataFrame API</a></li>
+<li class="toctree-l2"><a class="reference internal" href="spark-functions.html#sql">SQL</a></li>
+<li class="toctree-l2"><a class="reference internal" href="spark-functions.html#function-reference">Function Reference</a></li>
+</ul>
+</li>
<li class="toctree-l1"><a class="reference internal" href="aggregations.html">Aggregation</a><ul>
<li class="toctree-l2"><a class="reference internal" href="aggregations.html#setting-parameters">Setting Parameters</a></li>
<li class="toctree-l2"><a class="reference internal" href="aggregations.html#grouping-sets">Grouping Sets</a></li>
diff --git a/user-guide/common-operations/joins.html b/user-guide/common-operations/joins.html
index 8b1610c2..62dae6fb 100644
--- a/user-guide/common-operations/joins.html
+++ b/user-guide/common-operations/joins.html
@@ -364,6 +364,7 @@
<li class="toctree-l3 current active"><a class="current reference internal" href="#">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/common-operations/select-and-filter.html b/user-guide/common-operations/select-and-filter.html
index fa5302da..a8a225c3 100644
--- a/user-guide/common-operations/select-and-filter.html
+++ b/user-guide/common-operations/select-and-filter.html
@@ -360,6 +360,7 @@
<li class="toctree-l3"><a class="reference internal" href="joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="udf-and-udfa.html">User-Defined Functions</a></li>
@@ -393,7 +394,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/common-operations/views.html b/user-guide/common-operations/spark-functions.html
similarity index 70%
copy from user-guide/common-operations/views.html
copy to user-guide/common-operations/spark-functions.html
index c314468b..c2cc1283 100644
--- a/user-guide/common-operations/views.html
+++ b/user-guide/common-operations/spark-functions.html
@@ -8,7 +8,7 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
- <title>Registering Views — Apache DataFusion in Python documentation</title>
+ <title>Spark-Compatible Functions — Apache DataFusion in Python documentation</title>
@@ -44,13 +44,13 @@
<script src="../../_static/documentation_options.js?v=5929fcd5"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
- <script>DOCUMENTATION_OPTIONS.pagename = 'user-guide/common-operations/views';</script>
+ <script>DOCUMENTATION_OPTIONS.pagename = 'user-guide/common-operations/spark-functions';</script>
<script src="../../_static/toc-toggle.js?v=36375f01"></script>
<link rel="icon" href="../../_static/favicon.svg"/>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="Basic Operations" href="basic-info.html" />
- <link rel="prev" title="Common Operations" href="index.html" />
+ <link rel="next" title="Aggregation" href="aggregations.html" />
+ <link rel="prev" title="Functions" href="functions.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="" />
@@ -233,6 +233,10 @@
+ <button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page">
+ <span class="fa-solid fa-outdent"></span>
+ </button>
+
</div>
</header>
@@ -353,13 +357,14 @@
</ul>
</details></li>
<li class="toctree-l2 current active has-children"><a class="reference internal" href="index.html">Common Operations</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current">
-<li class="toctree-l3 current active"><a class="current reference internal" href="#">Registering Views</a></li>
+<li class="toctree-l3"><a class="reference internal" href="views.html">Registering Views</a></li>
<li class="toctree-l3"><a class="reference internal" href="basic-info.html">Basic Operations</a></li>
<li class="toctree-l3"><a class="reference internal" href="select-and-filter.html">Column Selections</a></li>
<li class="toctree-l3"><a class="reference internal" href="expressions.html">Expressions</a></li>
<li class="toctree-l3"><a class="reference internal" href="joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="functions.html">Functions</a></li>
+<li class="toctree-l3 current active"><a class="current reference internal" href="#">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="udf-and-udfa.html">User-Defined Functions</a></li>
@@ -393,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
@@ -465,7 +473,7 @@
<li class="breadcrumb-item"><a href="index.html" class="nav-link">Common Operations</a></li>
- <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Registering Views</span></li>
+ <li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Spark-Compatible Functions</span></li>
</ul>
</nav>
</div>
@@ -500,42 +508,73 @@
specific language governing permissions and limitations
under the License.
-->
-<section id="registering-views">
-<h1>Registering Views<a class="headerlink" href="#registering-views" title="Link to this heading">#</a></h1>
-<p>You can use the context’s <code class="docutils literal notranslate"><span class="pre">register_view</span></code> method to register a DataFrame as a view</p>
-<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span><span class="p">,</span> <span class="n">col</span><span class="p">,</span> <span class="n">literal</span>
+<section id="spark-compatible-functions">
+<span id="spark-functions"></span><h1>Spark-Compatible Functions<a class="headerlink" href="#spark-compatible-functions" title="Link to this heading">#</a></h1>
+<p>DataFusion ships Spark-compatible versions of a wide set of functions
+(string, math, datetime, hash, array, aggregate) through the upstream
+<code class="docutils literal notranslate"><span class="pre">datafusion-spark</span></code> crate. <code class="docutils literal notranslate"><span class="pre">datafusion-python</span></code> exposes these under
+<code class="docutils literal notranslate"><span class="pre">datafusion.functions.spark</span></code> for use from the DataFrame API, and via
+<code class="xref py py-meth docutils literal notranslate"><span class="pre">enable_spark_functions()</span></code> for use from
+SQL.</p>
+<section id="why-a-separate-namespace">
+<h2>Why a Separate Namespace?<a class="headerlink" href="#why-a-separate-namespace" title="Link to this heading">#</a></h2>
+<p>Several Spark functions share names with DataFusion built-ins but differ in
+semantics. The most common divergences:</p>
+<ul class="simple">
+<li><p><code class="docutils literal notranslate"><span class="pre">concat</span></code> propagates NULL. <code class="docutils literal notranslate"><span class="pre">concat('a',</span> <span class="pre">NULL,</span> <span class="pre">'b')</span></code> returns NULL under
+Spark semantics, whereas the DataFusion default returns <code class="docutils literal notranslate"><span class="pre">'ab'</span></code>.</p></li>
+<li><p><code class="docutils literal notranslate"><span class="pre">substring</span></code> is 1-indexed and supports negative positions counting from
+the end of the string.</p></li>
+<li><p><code class="docutils literal notranslate"><span class="pre">round</span></code> uses HALF_UP rounding mode (<code class="docutils literal notranslate"><span class="pre">round(2.5,</span> <span class="pre">0)</span> <span class="pre">==</span> <span class="pre">3</span></code>).</p></li>
+<li><p>Numeric functions (<code class="docutils literal notranslate"><span class="pre">floor</span></code>, <code class="docutils literal notranslate"><span class="pre">ceil</span></code>, <code class="docutils literal notranslate"><span class="pre">mod</span></code>) follow Spark’s edge-case
+handling for negative values and decimals.</p></li>
+</ul>
+<p>Enabling Spark functions does not affect the DataFrame API: you choose which
+implementation to call by which module you import from.</p>
+</section>
+<section id="dataframe-api">
+<h2>DataFrame API<a class="headerlink" href="#dataframe-api" title="Link to this heading">#</a></h2>
+<p>Import <code class="docutils literal notranslate"><span class="pre">spark</span></code> and use it like any other functions module. The Spark
+functions can go anywhere you’d put a DataFusion expression — inside
+<code class="docutils literal notranslate"><span class="pre">select</span></code>, <code class="docutils literal notranslate"><span class="pre">filter</span></code>, <code class="docutils literal notranslate"><span class="pre">with_column</span></code>, <code class="docutils literal notranslate"><span class="pre">aggregate</span></code>, and so on.</p>
+<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span><span class="p">,</span> <span class="n">col</span><span class="p">,</span> <span class="n">lit</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">datafusion.functions</span><span class="w"> </span><span class="kn">import</span> <span class="n">spark</span>
-<span class="c1"># Create a DataFusion context</span>
<span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"s"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"hello"</span><span class="p">,</span> <span class="s2">"world"</span><span class="p">]})</span>
-<span class="c1"># Create sample data</span>
-<span class="n">data</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p"> [...]
-
-<span class="c1"># Create a DataFrame from the dictionary</span>
-<span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="s2">"my_table"</span><span class="p">)</span>
+<span class="c1"># SHA-256 hash with Spark semantics</span>
+<span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">spark</span><span class="o">.</span><span class="n">sha2</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"s"</span><span class="p">),</span> <span class="n">lit</span><span class="p">(</span><span class="mi">256</span><span class="p">))</span><span class="o">.</span><span class="n">alias</span><span class="p">(</s [...]
-<span class="c1"># Filter the DataFrame (for example, keep rows where a > 2)</span>
-<span class="n">df_filtered</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span> <span class="o">></span> <span class="n">literal</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span>
-
-<span class="c1"># Register the dataframe as a view with the context</span>
-<span class="n">ctx</span><span class="o">.</span><span class="n">register_view</span><span class="p">(</span><span class="s2">"view1"</span><span class="p">,</span> <span class="n">df_filtered</span><span class="p">)</span>
-
-<span class="c1"># Now run a SQL query against the registered view</span>
-<span class="n">df_view</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT * FROM view1"</span><span class="p">)</span>
-
-<span class="c1"># Collect the results</span>
-<span class="n">results</span> <span class="o">=</span> <span class="n">df_view</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
-
-<span class="c1"># Convert results to a list of dictionaries for display</span>
-<span class="n">result_dicts</span> <span class="o">=</span> <span class="p">[</span><span class="n">batch</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span> <span class="k">for</span> <span class="n">batch</span> <span class="ow">in</span> <span class="n">results</span><span class="p">]</span>
-
-<span class="nb">print</span><span class="p">(</span><span class="n">result_dicts</span><span class="p">)</span>
+<span class="c1"># 1-indexed substring</span>
+<span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">spark</span><span class="o">.</span><span class="n">substring</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"s"</span><span class="p">),</span> <span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">lit</span><span class="p">(</span><span class="mi">3 [...]
</pre></div>
</div>
-<p>This will output:</p>
-<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="p">[{</span><span class="s1">'a'</span><span class="p">:</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">],</span> <span class="s1">'b'</span><span class="p">:</span> <span class="p">[</span><span class="mi">30</span><span class="p">,</span> <span class="m [...]
+</section>
+<section id="sql">
+<h2>SQL<a class="headerlink" href="#sql" title="Link to this heading">#</a></h2>
+<p>To use Spark functions in SQL queries, call
+<code class="xref py py-meth docutils literal notranslate"><span class="pre">enable_spark_functions()</span></code> on the context.
+This registers every Spark UDF/UDAF/UDWF, overriding any DataFusion built-in
+of the same name.</p>
+<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span>
+
+<span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="n">ctx</span><span class="o">.</span><span class="n">enable_spark_functions</span><span class="p">()</span>
+
+<span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT sha2('hello', 256)"</span><span class="p">)</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
+<span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT concat('a', NULL, 'b')"</span><span class="p">)</span><span class="o">.</span><span class="n">show</span><span class="p">()</span> <span class="c1"># -> NULL, not 'ab'</span>
</pre></div>
</div>
+<p>The override applies for the lifetime of the session. To call DataFusion’s
+built-in versions afterwards, create a fresh <code class="docutils literal notranslate"><span class="pre">SessionContext</span></code>.</p>
+</section>
+<section id="function-reference">
+<h2>Function Reference<a class="headerlink" href="#function-reference" title="Link to this heading">#</a></h2>
+<p>The full, up-to-date list of available Spark functions — with signatures
+and per-function docstrings — lives in the
+<a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html#module-datafusion.functions.spark" title="datafusion.functions.spark"><code class="xref py py-mod docutils literal notranslate"><span class="pre">datafusion.functions.spark</span></code></a> API reference.</p>
+</section>
</section>
@@ -549,20 +588,20 @@
<div class="prev-next-area">
<a class="left-prev"
- href="index.html"
+ href="functions.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Common Operations</p>
+ <p class="prev-next-title">Functions</p>
</div>
</a>
<a class="right-next"
- href="basic-info.html"
+ href="aggregations.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">Basic Operations</p>
+ <p class="prev-next-title">Aggregation</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
@@ -573,6 +612,27 @@
+ <dialog id="pst-secondary-sidebar-modal"></dialog>
+ <div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
+
+
+ <div class="sidebar-secondary-item">
+<div
+ id="pst-page-navigation-heading-2"
+ class="page-toc tocsection onthispage">
+ <i class="fa-solid fa-list"></i> On this page
+ </div>
+ <nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2">
+ <ul class="visible nav section-nav flex-column">
+<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#why-a-separate-namespace">Why a Separate Namespace?</a></li>
+<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#dataframe-api">DataFrame API</a></li>
+<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#sql">SQL</a></li>
+<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#function-reference">Function Reference</a></li>
+</ul>
+ </nav></div>
+
+</div></div>
+
</div>
<footer class="bd-footer-content">
diff --git a/user-guide/common-operations/udf-and-udfa.html b/user-guide/common-operations/udf-and-udfa.html
index c16b8111..65a985ce 100644
--- a/user-guide/common-operations/udf-and-udfa.html
+++ b/user-guide/common-operations/udf-and-udfa.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="windows.html">Window Functions</a></li>
<li class="toctree-l3 current active"><a class="current reference internal" href="#">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
@@ -671,7 +675,7 @@ over column references and literals, so the optimizer can analyze it:</p>
<div class="cell_output docutils container">
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>FilterExec: brand@1 = A AND qty@2 >= 150
RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
- DataSourceExec: file_groups={1 group: [[tmp/tmpziu60m3z/items.parquet]]}, projection=[id, brand, qty], file_type=parquet, predicate=brand@1 = A AND qty@2 >= 150, pruning_predicate=brand_null_count@2 != row_count@3 AND brand_min@0 <= A AND A <= brand_max@1 AND qty_null_count@5 != row_count@3 AND qty_max@4 >= 150, required_guarantees=[brand in (A)]
+ DataSourceExec: file_groups={1 group: [[tmp/tmpe03924tw/items.parquet]]}, projection=[id, brand, qty], file_type=parquet, predicate=brand@1 = A AND qty@2 >= 150, pruning_predicate=brand_null_count@2 != row_count@3 AND brand_min@0 <= A AND A <= brand_max@1 AND qty_null_count@5 != row_count@3 AND qty_max@4 >= 150, required_guarantees=[brand in (A)]
</pre></div>
</div>
</div>
@@ -707,7 +711,7 @@ bloom filter or dictionary is available to skip pages.</p></li>
<div class="cell_output docutils container">
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>FilterExec: brand_qty_filter(CAST(brand@1 AS Utf8), qty@2)
RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
- DataSourceExec: file_groups={1 group: [[tmp/tmpziu60m3z/items.parquet]]}, projection=[id, brand, qty], file_type=parquet, predicate=brand_qty_filter(CAST(brand@1 AS Utf8), qty@2)
+ DataSourceExec: file_groups={1 group: [[tmp/tmpe03924tw/items.parquet]]}, projection=[id, brand, qty], file_type=parquet, predicate=brand_qty_filter(CAST(brand@1 AS Utf8), qty@2)
</pre></div>
</div>
</div>
diff --git a/user-guide/common-operations/views.html b/user-guide/common-operations/views.html
index c314468b..e28d346d 100644
--- a/user-guide/common-operations/views.html
+++ b/user-guide/common-operations/views.html
@@ -360,6 +360,7 @@
<li class="toctree-l3"><a class="reference internal" href="joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="udf-and-udfa.html">User-Defined Functions</a></li>
@@ -393,7 +394,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/common-operations/windows.html b/user-guide/common-operations/windows.html
index abeb5bcb..b497f992 100644
--- a/user-guide/common-operations/windows.html
+++ b/user-guide/common-operations/windows.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="aggregations.html">Aggregation</a></li>
<li class="toctree-l3 current active"><a class="current reference internal" href="#">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/configuration.html b/user-guide/configuration.html
index c32c4869..e662837c 100644
--- a/user-guide/configuration.html
+++ b/user-guide/configuration.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/data-sources.html b/user-guide/data-sources.html
index 4da6ea54..eedf6694 100644
--- a/user-guide/data-sources.html
+++ b/user-guide/data-sources.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/dataframe/execution-metrics.html b/user-guide/dataframe/execution-metrics.html
index 6ed2b82d..1db98b34 100644
--- a/user-guide/dataframe/execution-metrics.html
+++ b/user-guide/dataframe/execution-metrics.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/dataframe/index.html b/user-guide/dataframe/index.html
index dfbe82d9..2a10237e 100644
--- a/user-guide/dataframe/index.html
+++ b/user-guide/dataframe/index.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/dataframe/rendering.html b/user-guide/dataframe/rendering.html
index 58e0de43..80ae9938 100644
--- a/user-guide/dataframe/rendering.html
+++ b/user-guide/dataframe/rendering.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/distributing-work.html b/user-guide/distributing-work.html
index 3e23c5bf..26e0f0de 100644
--- a/user-guide/distributing-work.html
+++ b/user-guide/distributing-work.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/index.html b/user-guide/index.html
index e0587eb9..57d5a76e 100644
--- a/user-guide/index.html
+++ b/user-guide/index.html
@@ -360,6 +360,7 @@
<li class="toctree-l3"><a class="reference internal" href="common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -393,7 +394,10 @@
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
@@ -547,6 +551,7 @@ with the DataFrame API or SQL, reading and writing data, and tuning execution.</
<li class="toctree-l2"><a class="reference internal" href="common-operations/joins.html">Joins</a></li>
<li class="toctree-l2"><a class="reference internal" href="common-operations/functions.html">Functions</a></li>
<li class="toctree-l2"><a class="reference internal" href="common-operations/functions.html#handling-missing-values">Handling Missing Values</a></li>
+<li class="toctree-l2"><a class="reference internal" href="common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l2"><a class="reference internal" href="common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l2"><a class="reference internal" href="common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l2"><a class="reference internal" href="common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
diff --git a/user-guide/introduction.html b/user-guide/introduction.html
index 4d12758e..f47eea27 100644
--- a/user-guide/introduction.html
+++ b/user-guide/introduction.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/io/arrow.html b/user-guide/io/arrow.html
index c3c53d7e..5cc0b8c9 100644
--- a/user-guide/io/arrow.html
+++ b/user-guide/io/arrow.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="../common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/io/avro.html b/user-guide/io/avro.html
index e432e144..036f5ebc 100644
--- a/user-guide/io/avro.html
+++ b/user-guide/io/avro.html
@@ -360,6 +360,7 @@
<li class="toctree-l3"><a class="reference internal" href="../common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -393,7 +394,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/io/csv.html b/user-guide/io/csv.html
index b8458898..e535195e 100644
--- a/user-guide/io/csv.html
+++ b/user-guide/io/csv.html
@@ -360,6 +360,7 @@
<li class="toctree-l3"><a class="reference internal" href="../common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -393,7 +394,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/io/index.html b/user-guide/io/index.html
index 01ff9813..7d00b6c6 100644
--- a/user-guide/io/index.html
+++ b/user-guide/io/index.html
@@ -360,6 +360,7 @@
<li class="toctree-l3"><a class="reference internal" href="../common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -393,7 +394,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/io/json.html b/user-guide/io/json.html
index c3711df6..d310163a 100644
--- a/user-guide/io/json.html
+++ b/user-guide/io/json.html
@@ -360,6 +360,7 @@
<li class="toctree-l3"><a class="reference internal" href="../common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -393,7 +394,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/io/parquet.html b/user-guide/io/parquet.html
index 89a15cd2..d44239de 100644
--- a/user-guide/io/parquet.html
+++ b/user-guide/io/parquet.html
@@ -360,6 +360,7 @@
<li class="toctree-l3"><a class="reference internal" href="../common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -393,7 +394,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/io/table_provider.html b/user-guide/io/table_provider.html
index f48324e5..6d06dd06 100644
--- a/user-guide/io/table_provider.html
+++ b/user-guide/io/table_provider.html
@@ -360,6 +360,7 @@
<li class="toctree-l3"><a class="reference internal" href="../common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -393,7 +394,10 @@
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/sql.html b/user-guide/sql.html
index a051b21c..f0db1ae5 100644
--- a/user-guide/sql.html
+++ b/user-guide/sql.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
diff --git a/user-guide/upgrade-guides.html b/user-guide/upgrade-guides.html
index 5fdfee60..8f2b6d23 100644
--- a/user-guide/upgrade-guides.html
+++ b/user-guide/upgrade-guides.html
@@ -364,6 +364,7 @@
<li class="toctree-l3"><a class="reference internal" href="common-operations/joins.html">Joins</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/functions.html">Functions</a></li>
+<li class="toctree-l3"><a class="reference internal" href="common-operations/spark-functions.html">Spark-Compatible Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/aggregations.html">Aggregation</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/windows.html">Window Functions</a></li>
<li class="toctree-l3"><a class="reference internal" href="common-operations/udf-and-udfa.html">User-Defined Functions</a></li>
@@ -397,7 +398,10 @@
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/expr/index.html">datafusion.expr</a></li>
-<li class="toctree-l3"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a></li>
+<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/functions/spark/index.html">datafusion.functions.spark</a></li>
+</ul>
+</details></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../autoapi/datafusion/input/index.html">datafusion.input</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/base/index.html">datafusion.input.base</a></li>
<li class="toctree-l4"><a class="reference internal" href="../autoapi/datafusion/input/location/index.html">datafusion.input.location</a></li>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org