You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/04/09 12:52:37 UTC
(datafusion-python) branch asf-site updated: Publish built docs triggered by af383af369ec442e3b88841df3ba791b3a383413
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/datafusion-python.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 16620b0f Publish built docs triggered by af383af369ec442e3b88841df3ba791b3a383413
16620b0f is described below
commit 16620b0fb4b61f6626d1bb9134d1a7dde74c885e
Author: github-actions[bot] <gi...@users.noreply.github.com>
AuthorDate: Thu Apr 9 12:52:32 2026 +0000
Publish built docs triggered by af383af369ec442e3b88841df3ba791b3a383413
---
_sources/autoapi/datafusion/catalog/index.rst.txt | 6 -
_sources/autoapi/datafusion/context/index.rst.txt | 383 +-
.../autoapi/datafusion/dataframe/index.rst.txt | 459 +-
.../datafusion/dataframe_formatter/index.rst.txt | 8 +-
_sources/autoapi/datafusion/expr/index.rst.txt | 165 +-
.../autoapi/datafusion/functions/index.rst.txt | 3524 ++++++++++++++-
.../datafusion/html_formatter/index.rst.txt | 11 -
_sources/autoapi/datafusion/index.rst.txt | 568 +--
.../autoapi/datafusion/user_defined/index.rst.txt | 203 +-
_sources/contributor-guide/ffi.rst.txt | 13 +-
.../common-operations/aggregations.rst.txt | 172 +-
.../user-guide/common-operations/joins.rst.txt | 33 +
.../user-guide/common-operations/windows.rst.txt | 15 +-
_sources/user-guide/data-sources.rst.txt | 26 +-
_sources/user-guide/dataframe/rendering.rst.txt | 225 +-
_sources/user-guide/upgrade-guides.rst.txt | 27 +-
autoapi/datafusion/catalog/index.html | 22 +-
autoapi/datafusion/context/index.html | 632 ++-
autoapi/datafusion/dataframe/index.html | 610 ++-
autoapi/datafusion/dataframe_formatter/index.html | 15 +-
autoapi/datafusion/expr/index.html | 329 +-
autoapi/datafusion/functions/index.html | 4617 +++++++++++++++++++-
autoapi/datafusion/html_formatter/index.html | 496 ---
autoapi/datafusion/index.html | 1034 +----
autoapi/datafusion/input/base/index.html | 7 +-
autoapi/datafusion/input/index.html | 13 +-
autoapi/datafusion/input/location/index.html | 7 +-
autoapi/datafusion/io/index.html | 7 +-
autoapi/datafusion/object_store/index.html | 7 +-
autoapi/datafusion/options/index.html | 7 +-
autoapi/datafusion/plan/index.html | 7 +-
autoapi/datafusion/record_batch/index.html | 7 +-
autoapi/datafusion/substrait/index.html | 7 +-
autoapi/datafusion/unparser/index.html | 7 +-
autoapi/datafusion/user_defined/index.html | 194 +-
autoapi/index.html | 8 +-
contributor-guide/ffi.html | 20 +-
contributor-guide/introduction.html | 7 +-
genindex.html | 476 +-
index.html | 7 +-
objects.inv | Bin 9481 -> 9769 bytes
py-modindex.html | 12 +-
search.html | 7 +-
searchindex.js | 2 +-
user-guide/basics.html | 7 +-
user-guide/common-operations/aggregations.html | 328 +-
user-guide/common-operations/basic-info.html | 41 +-
user-guide/common-operations/expressions.html | 7 +-
user-guide/common-operations/functions.html | 27 +-
user-guide/common-operations/index.html | 13 +-
user-guide/common-operations/joins.html | 59 +-
.../common-operations/select-and-filter.html | 7 +-
user-guide/common-operations/udf-and-udfa.html | 7 +-
user-guide/common-operations/views.html | 7 +-
user-guide/common-operations/windows.html | 22 +-
user-guide/configuration.html | 7 +-
user-guide/data-sources.html | 36 +-
user-guide/dataframe/index.html | 15 +-
user-guide/dataframe/rendering.html | 265 +-
user-guide/introduction.html | 9 +-
user-guide/io/arrow.html | 7 +-
user-guide/io/avro.html | 7 +-
user-guide/io/csv.html | 7 +-
user-guide/io/index.html | 7 +-
user-guide/io/json.html | 7 +-
user-guide/io/parquet.html | 7 +-
user-guide/io/table_provider.html | 7 +-
user-guide/sql.html | 7 +-
user-guide/upgrade-guides.html | 34 +-
69 files changed, 11767 insertions(+), 3582 deletions(-)
diff --git a/_sources/autoapi/datafusion/catalog/index.rst.txt b/_sources/autoapi/datafusion/catalog/index.rst.txt
index f2c9776f..dcd21725 100644
--- a/_sources/autoapi/datafusion/catalog/index.rst.txt
+++ b/_sources/autoapi/datafusion/catalog/index.rst.txt
@@ -39,12 +39,6 @@ Module Contents
- .. py:method:: database(name: str = 'public') -> Schema
-
- Returns the database with the given ``name`` from this catalog.
-
-
-
.. py:method:: deregister_schema(name: str, cascade: bool = True) -> Schema | None
Deregister a schema from this catalog.
diff --git a/_sources/autoapi/datafusion/context/index.rst.txt b/_sources/autoapi/datafusion/context/index.rst.txt
index f010a010..a0a5fc8d 100644
--- a/_sources/autoapi/datafusion/context/index.rst.txt
+++ b/_sources/autoapi/datafusion/context/index.rst.txt
@@ -16,7 +16,6 @@ Classes
datafusion.context.ArrowArrayExportable
datafusion.context.ArrowStreamExportable
- datafusion.context.RuntimeConfig
datafusion.context.RuntimeEnvBuilder
datafusion.context.SQLOptions
datafusion.context.SessionConfig
@@ -53,16 +52,6 @@ Module Contents
.. py:method:: __arrow_c_stream__(requested_schema: object | None = None) -> object
-.. py:class:: RuntimeConfig
-
- Bases: :py:obj:`RuntimeEnvBuilder`
-
-
- See `RuntimeEnvBuilder`.
-
- Create a new :py:class:`RuntimeEnvBuilder` with default values.
-
-
.. py:class:: RuntimeEnvBuilder
Runtime configuration options.
@@ -118,9 +107,9 @@ Module Contents
:returns: A new :py:class:`RuntimeEnvBuilder` object with the updated setting.
- Examples usage::
+ .. rubric:: Examples
- config = RuntimeEnvBuilder().with_fair_spill_pool(1024)
+ >>> config = dfn.RuntimeEnvBuilder().with_fair_spill_pool(1024)
@@ -136,9 +125,9 @@ Module Contents
:returns: A new :py:class:`RuntimeEnvBuilder` object with the updated setting.
- Example usage::
+ .. rubric:: Examples
- config = RuntimeEnvBuilder().with_greedy_memory_pool(1024)
+ >>> config = dfn.RuntimeEnvBuilder().with_greedy_memory_pool(1024)
@@ -150,9 +139,9 @@ Module Contents
:returns: A new :py:class:`RuntimeEnvBuilder` object with the updated setting.
- Example usage::
+ .. rubric:: Examples
- config = RuntimeEnvBuilder().with_temp_file_path("/tmp")
+ >>> config = dfn.RuntimeEnvBuilder().with_temp_file_path("/tmp")
@@ -189,9 +178,9 @@ Module Contents
:returns: A new :py:class:`SQLOptions` object with the updated setting.
- Example usage::
+ .. rubric:: Examples
- options = SQLOptions().with_allow_ddl(True)
+ >>> options = dfn.SQLOptions().with_allow_ddl(True)
@@ -205,9 +194,9 @@ Module Contents
:returns: A new :py:class:`SQLOptions` object with the updated setting.
- Example usage::
+ .. rubric:: Examples
- options = SQLOptions().with_allow_dml(True)
+ >>> options = dfn.SQLOptions().with_allow_dml(True)
@@ -220,9 +209,9 @@ Module Contents
:returns: py:class:SQLOptions` object with the updated setting.
:rtype: A new
- Example usage::
+ .. rubric:: Examples
- options = SQLOptions().with_allow_statements(True)
+ >>> options = dfn.SQLOptions().with_allow_statements(True)
@@ -282,6 +271,17 @@ Module Contents
+ .. py:method:: with_extension(extension: Any) -> SessionConfig
+
+ Create a new configuration using an extension.
+
+ :param extension: A custom configuration extension object. These are
+ :param shared from another DataFusion extension library.:
+
+ :returns: A new :py:class:`SessionConfig` object with the updated setting.
+
+
+
.. py:method:: with_information_schema(enabled: bool = True) -> SessionConfig
Enable or disable the inclusion of ``information_schema`` virtual tables.
@@ -480,18 +480,71 @@ Module Contents
+ .. py:method:: deregister_object_store(schema: str, host: str | None = None) -> None
+
+ Remove an object store from the session.
+
+ :param schema: The data source schema (e.g. ``"s3://"``).
+ :param host: URL for the host (e.g. bucket name).
+
+
+
.. py:method:: deregister_table(name: str) -> None
Remove a table from the session.
+ .. py:method:: deregister_udaf(name: str) -> None
+
+ Remove a user-defined aggregate function from the session.
+
+ :param name: Name of the UDAF to deregister.
+
+
+
+ .. py:method:: deregister_udf(name: str) -> None
+
+ Remove a user-defined scalar function from the session.
+
+ :param name: Name of the UDF to deregister.
+
+
+
+ .. py:method:: deregister_udtf(name: str) -> None
+
+ Remove a user-defined table function from the session.
+
+ :param name: Name of the UDTF to deregister.
+
+
+
+ .. py:method:: deregister_udwf(name: str) -> None
+
+ Remove a user-defined window function from the session.
+
+ :param name: Name of the UDWF to deregister.
+
+
+
.. py:method:: empty_table() -> datafusion.dataframe.DataFrame
Create an empty :py:class:`~datafusion.dataframe.DataFrame`.
+ .. py:method:: enable_ident_normalization() -> bool
+
+ Return whether identifier normalization (lowercasing) is enabled.
+
+ .. rubric:: Examples
+
+ >>> ctx = SessionContext()
+ >>> ctx.enable_ident_normalization()
+ True
+
+
+
.. py:method:: enable_url_table() -> SessionContext
Control if local files can be queried as tables.
@@ -506,6 +559,30 @@ Module Contents
+ .. py:method:: execute_logical_plan(plan: datafusion.plan.LogicalPlan) -> datafusion.dataframe.DataFrame
+
+ Execute a :py:class:`~datafusion.plan.LogicalPlan` and return a DataFrame.
+
+ :param plan: Logical plan to execute.
+
+ :returns: DataFrame resulting from the execution.
+
+ .. rubric:: Examples
+
+ >>> ctx = SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> plan = df.logical_plan()
+ >>> df2 = ctx.execute_logical_plan(plan)
+ >>> df2.collect()[0].column(0)
+ <pyarrow.lib.Int64Array object at ...>
+ [
+ 1,
+ 2,
+ 3
+ ]
+
+
+
.. py:method:: from_arrow(data: ArrowStreamExportable | ArrowArrayExportable, name: str | None = None) -> datafusion.dataframe.DataFrame
Create a :py:class:`~datafusion.dataframe.DataFrame` from an Arrow source.
@@ -523,14 +600,6 @@ Module Contents
- .. py:method:: from_arrow_table(data: pyarrow.Table, name: str | None = None) -> datafusion.dataframe.DataFrame
-
- Create a :py:class:`~datafusion.dataframe.DataFrame` from an Arrow table.
-
- This is an alias for :py:func:`from_arrow`.
-
-
-
.. py:method:: from_pandas(data: pandas.DataFrame, name: str | None = None) -> datafusion.dataframe.DataFrame
Create a :py:class:`~datafusion.dataframe.DataFrame` from a Pandas DataFrame.
@@ -585,6 +654,86 @@ Module Contents
+ .. py:method:: parse_sql_expr(sql: str, schema: datafusion.common.DFSchema) -> datafusion.expr.Expr
+
+ Parse a SQL expression string into a logical expression.
+
+ :param sql: SQL expression string.
+ :param schema: Schema to use for resolving column references.
+
+ :returns: Parsed expression.
+
+ .. rubric:: Examples
+
+ >>> from datafusion.common import DFSchema
+ >>> ctx = SessionContext()
+ >>> schema = DFSchema.empty()
+ >>> ctx.parse_sql_expr("1 + 2", schema=schema)
+ Expr(Int64(1) + Int64(2))
+
+
+
+ .. py:method:: read_arrow(path: str | pathlib.Path, schema: pyarrow.Schema | None = None, file_extension: str = '.arrow', file_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None) -> datafusion.dataframe.DataFrame
+
+ Create a :py:class:`DataFrame` for reading an Arrow IPC data source.
+
+ :param path: Path to the Arrow IPC file.
+ :param schema: The data source schema.
+ :param file_extension: File extension to select.
+ :param file_partition_cols: Partition columns.
+
+ :returns: DataFrame representation of the read Arrow IPC file.
+
+ .. rubric:: Examples
+
+ >>> import tempfile, os
+ >>> ctx = dfn.SessionContext()
+ >>> table = pa.table({"a": [1, 2, 3]})
+ >>> with tempfile.TemporaryDirectory() as tmpdir:
+ ... path = os.path.join(tmpdir, "data.arrow")
+ ... with pa.ipc.new_file(path, table.schema) as writer:
+ ... writer.write_table(table)
+ ... df = ctx.read_arrow(path)
+ ... df.collect()[0].column(0)
+ <pyarrow.lib.Int64Array object at ...>
+ [
+ 1,
+ 2,
+ 3
+ ]
+
+ Provide an explicit ``schema`` to override schema inference:
+
+ >>> with tempfile.TemporaryDirectory() as tmpdir:
+ ... path = os.path.join(tmpdir, "data.arrow")
+ ... with pa.ipc.new_file(path, table.schema) as writer:
+ ... writer.write_table(table)
+ ... df = ctx.read_arrow(path, schema=pa.schema([("a", pa.int64())]))
+ ... df.collect()[0].column(0)
+ <pyarrow.lib.Int64Array object at ...>
+ [
+ 1,
+ 2,
+ 3
+ ]
+
+ Use ``file_extension`` to read files with a non-default extension:
+
+ >>> with tempfile.TemporaryDirectory() as tmpdir:
+ ... path = os.path.join(tmpdir, "data.ipc")
+ ... with pa.ipc.new_file(path, table.schema) as writer:
+ ... writer.write_table(table)
+ ... df = ctx.read_arrow(path, file_extension=".ipc")
+ ... df.collect()[0].column(0)
+ <pyarrow.lib.Int64Array object at ...>
+ [
+ 1,
+ 2,
+ 3
+ ]
+
+
+
.. py:method:: read_avro(path: str | pathlib.Path, schema: pyarrow.Schema | None = None, file_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, file_extension: str = '.avro') -> datafusion.dataframe.DataFrame
Create a :py:class:`DataFrame` for reading Avro data source.
@@ -622,6 +771,14 @@ Module Contents
+ .. py:method:: read_empty() -> datafusion.dataframe.DataFrame
+
+ Create an empty :py:class:`DataFrame` with no columns or rows.
+
+ .. seealso:: This is an alias for :meth:`empty_table`.
+
+
+
.. py:method:: read_json(path: str | pathlib.Path, schema: pyarrow.Schema | None = None, schema_infer_max_records: int = 1000, file_extension: str = '.json', table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, file_compression_type: str | None = None) -> datafusion.dataframe.DataFrame
Read a line-delimited JSON data source.
@@ -669,6 +826,86 @@ Module Contents
+ .. py:method:: refresh_catalogs() -> None
+
+ Refresh catalog metadata.
+
+ .. rubric:: Examples
+
+ >>> ctx = SessionContext()
+ >>> ctx.refresh_catalogs()
+
+
+
+ .. py:method:: register_arrow(name: str, path: str | pathlib.Path, schema: pyarrow.Schema | None = None, file_extension: str = '.arrow', table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None) -> None
+
+ Register an Arrow IPC file as a table.
+
+ The registered table can be referenced from SQL statements executed
+ against this context.
+
+ :param name: Name of the table to register.
+ :param path: Path to the Arrow IPC file.
+ :param schema: The data source schema.
+ :param file_extension: File extension to select.
+ :param table_partition_cols: Partition columns.
+
+ .. rubric:: Examples
+
+ >>> import tempfile, os
+ >>> ctx = dfn.SessionContext()
+ >>> table = pa.table({"x": [10, 20, 30]})
+ >>> with tempfile.TemporaryDirectory() as tmpdir:
+ ... path = os.path.join(tmpdir, "data.arrow")
+ ... with pa.ipc.new_file(path, table.schema) as writer:
+ ... writer.write_table(table)
+ ... ctx.register_arrow("arrow_tbl", path)
+ ... ctx.sql("SELECT * FROM arrow_tbl").collect()[0].column(0)
+ <pyarrow.lib.Int64Array object at ...>
+ [
+ 10,
+ 20,
+ 30
+ ]
+
+ Provide an explicit ``schema`` to override schema inference:
+
+ >>> with tempfile.TemporaryDirectory() as tmpdir:
+ ... path = os.path.join(tmpdir, "data.arrow")
+ ... with pa.ipc.new_file(path, table.schema) as writer:
+ ... writer.write_table(table)
+ ... ctx.register_arrow(
+ ... "arrow_schema",
+ ... path,
+ ... schema=pa.schema([("x", pa.int64())]),
+ ... )
+ ... ctx.sql("SELECT * FROM arrow_schema").collect()[0].column(0)
+ <pyarrow.lib.Int64Array object at ...>
+ [
+ 10,
+ 20,
+ 30
+ ]
+
+ Use ``file_extension`` to read files with a non-default extension:
+
+ >>> with tempfile.TemporaryDirectory() as tmpdir:
+ ... path = os.path.join(tmpdir, "data.ipc")
+ ... with pa.ipc.new_file(path, table.schema) as writer:
+ ... writer.write_table(table)
+ ... ctx.register_arrow(
+ ... "arrow_ipc", path, file_extension=".ipc"
+ ... )
+ ... ctx.sql("SELECT * FROM arrow_ipc").collect()[0].column(0)
+ <pyarrow.lib.Int64Array object at ...>
+ [
+ 10,
+ 20,
+ 30
+ ]
+
+
+
.. py:method:: register_avro(name: str, path: str | pathlib.Path, schema: pyarrow.Schema | None = None, file_extension: str = '.avro', table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None) -> None
Register an Avro file as a table.
@@ -684,6 +921,28 @@ Module Contents
+ .. py:method:: register_batch(name: str, batch: pyarrow.RecordBatch) -> None
+
+ Register a single :py:class:`pa.RecordBatch` as a table.
+
+ :param name: Name of the resultant table.
+ :param batch: Record batch to register as a table.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> batch = pa.RecordBatch.from_pydict({"a": [1, 2, 3]})
+ >>> ctx.register_batch("batch_tbl", batch)
+ >>> ctx.sql("SELECT * FROM batch_tbl").collect()[0].column(0)
+ <pyarrow.lib.Int64Array object at ...>
+ [
+ 1,
+ 2,
+ 3
+ ]
+
+
+
.. py:method:: register_catalog_provider(name: str, provider: datafusion.catalog.CatalogProviderExportable | datafusion.catalog.CatalogProvider | datafusion.catalog.Catalog) -> None
Register a catalog provider.
@@ -825,6 +1084,18 @@ Module Contents
+ .. py:method:: register_table_factory(format: str, factory: datafusion.catalog.TableProviderFactory | datafusion.catalog.TableProviderFactoryExportable) -> None
+
+ Register a :py:class:`~datafusion.TableProviderFactoryExportable`.
+
+ The registered factory can be referenced from SQL DDL statements executed
+ against this context.
+
+ :param format: The value to be used in `STORED AS ${format}` clause.
+ :param factory: A PyCapsule that implements :class:`TableProviderFactoryExportable`
+
+
+
.. py:method:: register_table_provider(name: str, provider: datafusion.catalog.Table | TableProviderExportable | datafusion.dataframe.DataFrame | pyarrow.dataset.Dataset) -> None
Register a table provider.
@@ -868,12 +1139,40 @@ Module Contents
+ .. py:method:: remove_optimizer_rule(name: str) -> bool
+
+ Remove an optimizer rule by name.
+
+ :param name: Name of the optimizer rule to remove.
+
+ :returns: True if a rule with the given name was found and removed.
+
+ .. rubric:: Examples
+
+ >>> ctx = SessionContext()
+ >>> ctx.remove_optimizer_rule("nonexistent_rule")
+ False
+
+
+
.. py:method:: session_id() -> str
Return an id that uniquely identifies this :py:class:`SessionContext`.
+ .. py:method:: session_start_time() -> str
+
+ Return the session start time as an RFC 3339 formatted string.
+
+ .. rubric:: Examples
+
+ >>> ctx = SessionContext()
+ >>> ctx.session_start_time() # doctest: +SKIP
+ '2026-01-01T12:34:56.123456789+00:00'
+
+
+
.. py:method:: sql(query: str, options: SQLOptions | None = None, param_values: dict[str, Any] | None = None, **named_params: Any) -> datafusion.dataframe.DataFrame
Create a :py:class:`~datafusion.DataFrame` from SQL query text.
@@ -926,6 +1225,28 @@ Module Contents
+ .. py:method:: table_provider(name: str) -> datafusion.catalog.Table
+
+ Return the :py:class:`~datafusion.catalog.Table` for the given table name.
+
+ :param name: Name of the table.
+
+ :returns: The table provider.
+
+ :raises KeyError: If the table is not found.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> ctx = SessionContext()
+ >>> batch = pa.RecordBatch.from_pydict({"x": [1, 2]})
+ >>> ctx.register_record_batches("my_table", [[batch]])
+ >>> tbl = ctx.table_provider("my_table")
+ >>> tbl.schema
+ x: int64
+
+
+
.. py:method:: with_logical_extension_codec(codec: Any) -> SessionContext
Create a new session context with specified codec.
diff --git a/_sources/autoapi/datafusion/dataframe/index.rst.txt b/_sources/autoapi/datafusion/dataframe/index.rst.txt
index 0ac97106..900e339b 100644
--- a/_sources/autoapi/datafusion/dataframe/index.rst.txt
+++ b/_sources/autoapi/datafusion/dataframe/index.rst.txt
@@ -19,6 +19,7 @@ Classes
datafusion.dataframe.Compression
datafusion.dataframe.DataFrame
datafusion.dataframe.DataFrameWriteOptions
+ datafusion.dataframe.ExplainFormat
datafusion.dataframe.InsertOp
datafusion.dataframe.ParquetColumnOptions
datafusion.dataframe.ParquetWriterOptions
@@ -181,7 +182,21 @@ Module Contents
Aggregates the rows of the current DataFrame.
- :param group_by: Sequence of expressions or column names to group by.
+ By default each unique combination of the ``group_by`` columns
+ produces one row. To get multiple levels of subtotals in a
+ single pass, pass a
+ :py:class:`~datafusion.expr.GroupingSet` expression
+ (created via
+ :py:meth:`~datafusion.expr.GroupingSet.rollup`,
+ :py:meth:`~datafusion.expr.GroupingSet.cube`, or
+ :py:meth:`~datafusion.expr.GroupingSet.grouping_sets`)
+ as the ``group_by`` argument. See the
+ :ref:`aggregation` user guide for detailed examples.
+
+ :param group_by: Sequence of expressions or column names to group
+ by. A :py:class:`~datafusion.expr.GroupingSet`
+ expression may be included to produce multiple grouping
+ levels (rollup, cube, or explicit grouping sets).
:param aggs: Sequence of expressions to aggregate.
:returns: DataFrame after aggregation.
@@ -206,6 +221,14 @@ Module Contents
+ .. py:method:: col(name: str) -> datafusion.expr.Expr
+
+ Alias for :py:meth:`column`.
+
+ .. seealso:: :py:meth:`column`
+
+
+
.. py:method:: collect() -> list[pyarrow.RecordBatch]
Execute this :py:class:`DataFrame` and collect results into memory.
@@ -238,6 +261,43 @@ Module Contents
+ .. py:method:: column(name: str) -> datafusion.expr.Expr
+
+ Return a fully qualified column expression for ``name``.
+
+ Resolves an unqualified column name against this DataFrame's schema
+ and returns an :py:class:`Expr` whose underlying column reference
+ includes the table qualifier. This is especially useful after joins,
+ where the same column name may appear in multiple relations.
+
+ :param name: Unqualified column name to look up.
+
+ :returns: A fully qualified column expression.
+
+ :raises Exception: If the column is not found or is ambiguous (exists in
+ multiple relations).
+
+ .. rubric:: Examples
+
+ Resolve a column from a simple DataFrame:
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2], "b": [3, 4]})
+ >>> expr = df.column("a")
+ >>> df.select(expr).to_pydict()
+ {'a': [1, 2]}
+
+ Resolve qualified columns after a join:
+
+ >>> left = ctx.from_pydict({"id": [1, 2], "x": [10, 20]})
+ >>> right = ctx.from_pydict({"id": [1, 2], "y": [30, 40]})
+ >>> joined = left.join(right, on="id", how="inner")
+ >>> expr = joined.column("y")
+ >>> joined.select("id", expr).sort("id").to_pydict()
+ {'id': [1, 2], 'y': [30, 40]}
+
+
+
.. py:method:: count() -> int
Return the total number of rows in this :py:class:`DataFrame`.
@@ -281,35 +341,92 @@ Module Contents
+ .. py:method:: distinct_on(on_expr: list[datafusion.expr.Expr], select_expr: list[datafusion.expr.Expr], sort_expr: list[datafusion.expr.SortKey] | None = None) -> DataFrame
+
+ Deduplicate rows based on specific columns.
+
+ Returns a new DataFrame with one row per unique combination of the
+ ``on_expr`` columns, keeping the first row per group as determined by
+ ``sort_expr``.
+
+ :param on_expr: Expressions that determine uniqueness.
+ :param select_expr: Expressions to include in the output.
+ :param sort_expr: Optional sort expressions to determine which row to keep.
+
+ :returns: DataFrame after deduplication.
+
+ .. rubric:: Examples
+
+ Keep the row with the smallest ``b`` for each unique ``a``:
+
+ >>> from datafusion import col
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 1, 2, 2], "b": [10, 20, 30, 40]})
+ >>> df.distinct_on(
+ ... [col("a")],
+ ... [col("a"), col("b")],
+ ... [col("a").sort(ascending=True), col("b").sort(ascending=True)],
+ ... ).sort("a").to_pydict()
+ {'a': [1, 2], 'b': [10, 30]}
+
+
+
.. py:method:: drop(*columns: str) -> DataFrame
Drop arbitrary amount of columns.
- Column names are case-sensitive and do not require double quotes like
- other operations such as `select`. Leading and trailing double quotes
- are allowed and will be automatically stripped if present.
+ Column names are case-sensitive and require double quotes to be dropped
+ if the original name is not strictly lower case.
- :param columns: Column names to drop from the dataframe. Both ``column_name``
- and ``"column_name"`` are accepted.
+ :param columns: Column names to drop from the dataframe.
:returns: DataFrame with those columns removed in the projection.
- Example Usage::
+ .. rubric:: Examples
+
+ To drop a lower-cased column 'a'
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2], "b": [3, 4]})
+ >>> df.drop("a").schema().names
+ ['b']
+
+ Or to drop an upper-cased column 'A'
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"A": [1, 2], "b": [3, 4]})
+ >>> df.drop('"A"').schema().names
+ ['b']
- df.drop('ID_For_Students') # Works
- df.drop('"ID_For_Students"') # Also works (quotes stripped)
+ .. py:method:: except_all(other: DataFrame, distinct: bool = False) -> DataFrame
- .. py:method:: except_all(other: DataFrame) -> DataFrame
+ Calculate the set difference of two :py:class:`DataFrame`.
- Calculate the exception of two :py:class:`DataFrame`.
+ Returns rows that are in this DataFrame but not in ``other``.
The two :py:class:`DataFrame` must have exactly the same schema.
:param other: DataFrame to calculate exception with.
+ :param distinct: If ``True``, duplicate rows are removed from the result.
+
+ :returns: DataFrame after set difference.
+
+ .. rubric:: Examples
+
+ Remove rows present in ``df2``:
+
+ >>> ctx = dfn.SessionContext()
+ >>> df1 = ctx.from_pydict({"a": [1, 2, 3], "b": [10, 20, 30]})
+ >>> df2 = ctx.from_pydict({"a": [1, 2], "b": [10, 20]})
+ >>> df1.except_all(df2).sort("a").to_pydict()
+ {'a': [3], 'b': [30]}
- :returns: DataFrame after exception.
+ Remove rows present in ``df2`` and deduplicate:
+
+ >>> df1.except_all(df2, distinct=True).sort("a").to_pydict()
+ {'a': [3], 'b': [30]}
@@ -337,7 +454,7 @@ Module Contents
- .. py:method:: explain(verbose: bool = False, analyze: bool = False) -> None
+ .. py:method:: explain(verbose: bool = False, analyze: bool = False, format: ExplainFormat | None = None) -> None
Print an explanation of the DataFrame's plan so far.
@@ -345,6 +462,21 @@ Module Contents
:param verbose: If ``True``, more details will be included.
:param analyze: If ``True``, the plan will run and metrics reported.
+ :param format: Output format for the plan. Defaults to
+ :py:attr:`ExplainFormat.INDENT`.
+
+ .. rubric:: Examples
+
+ Show the plan in tree format:
+
+ >>> from datafusion import ExplainFormat
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> df.explain(format=ExplainFormat.TREE) # doctest: +SKIP
+
+ Show plan with runtime metrics:
+
+ >>> df.explain(analyze=True) # doctest: +SKIP
@@ -359,9 +491,12 @@ Module Contents
.. rubric:: Examples
- >>> df = df.fill_null(0) # Fill all nulls with 0 where possible
- >>> # Fill nulls in specific string columns
- >>> df = df.fill_null("missing", subset=["name", "category"])
+ >>> from datafusion import SessionContext, col
+ >>> ctx = SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, None, 3], "b": [None, 5, 6]})
+ >>> filled = df.fill_null(0)
+ >>> filled.sort(col("a")).collect()[0].column("a").to_pylist()
+ [0, 1, 3]
.. rubric:: Notes
@@ -383,11 +518,14 @@ Module Contents
that will be parsed against the DataFrame schema. If more complex logic is
required, see the logical operations in :py:mod:`~datafusion.functions`.
- Example::
+ .. rubric:: Examples
- from datafusion import col, lit
- df.filter(col("a") > lit(1))
- df.filter("a > 1")
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> df.filter(col("a") > lit(1)).to_pydict()
+ {'a': [2, 3]}
+ >>> df.filter("a > 1").to_pydict()
+ {'a': [2, 3]}
:param predicates: Predicate expression(s) or SQL strings to filter the DataFrame.
@@ -395,6 +533,32 @@ Module Contents
+ .. py:method:: find_qualified_columns(*names: str) -> list[datafusion.expr.Expr]
+
+ Return fully qualified column expressions for the given names.
+
+ This is a batch version of :py:meth:`column` — it resolves each
+ unqualified name against the DataFrame's schema and returns a list
+ of qualified column expressions.
+
+ :param names: Unqualified column names to look up.
+
+ :returns: List of fully qualified column expressions, one per name.
+
+ :raises Exception: If any column is not found or is ambiguous.
+
+ .. rubric:: Examples
+
+ Resolve multiple columns at once:
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2], "b": [3, 4], "c": [5, 6]})
+ >>> exprs = df.find_qualified_columns("a", "c")
+ >>> df.select(*exprs).to_pydict()
+ {'a': [1, 2], 'c': [5, 6]}
+
+
+
.. py:method:: head(n: int = 5) -> DataFrame
Return a new :py:class:`DataFrame` with a limited number of rows.
@@ -405,16 +569,34 @@ Module Contents
- .. py:method:: intersect(other: DataFrame) -> DataFrame
+ .. py:method:: intersect(other: DataFrame, distinct: bool = False) -> DataFrame
Calculate the intersection of two :py:class:`DataFrame`.
The two :py:class:`DataFrame` must have exactly the same schema.
:param other: DataFrame to intersect with.
+ :param distinct: If ``True``, duplicate rows are removed from the result.
:returns: DataFrame after intersection.
+ .. rubric:: Examples
+
+ Find rows common to both DataFrames:
+
+ >>> ctx = dfn.SessionContext()
+ >>> df1 = ctx.from_pydict({"a": [1, 2, 3], "b": [10, 20, 30]})
+ >>> df2 = ctx.from_pydict({"a": [1, 4], "b": [10, 40]})
+ >>> df1.intersect(df2).to_pydict()
+ {'a': [1], 'b': [10]}
+
+ Intersect with deduplication:
+
+ >>> df1 = ctx.from_pydict({"a": [1, 1, 2], "b": [10, 10, 20]})
+ >>> df2 = ctx.from_pydict({"a": [1, 1], "b": [10, 10]})
+ >>> df1.intersect(df2, distinct=True).to_pydict()
+ {'a': [1], 'b': [10]}
+
.. py:method:: into_view(temporary: bool = False) -> datafusion.catalog.Table
@@ -428,8 +610,9 @@ Module Contents
>>> df = ctx.sql("SELECT 1 AS value")
>>> view = df.into_view()
>>> ctx.register_table("values_view", view)
- >>> df.collect() # The DataFrame is still usable
- >>> ctx.sql("SELECT value FROM values_view").collect()
+ >>> result = ctx.sql("SELECT value FROM values_view").collect()
+ >>> result[0].column("value").to_pylist()
+ [1]
@@ -439,7 +622,13 @@ Module Contents
Join this :py:class:`DataFrame` with another :py:class:`DataFrame`.
- `on` has to be provided or both `left_on` and `right_on` in conjunction.
+ ``on`` has to be provided or both ``left_on`` and ``right_on`` in
+ conjunction.
+
+ When non-key columns share the same name in both DataFrames, use
+ :py:meth:`DataFrame.col` on each DataFrame **before** the join to
+ obtain fully qualified column references that can disambiguate them.
+ See :py:meth:`join_on` for an example.
:param right: Other DataFrame to join with.
:param on: Column names to join on in both dataframes.
@@ -464,10 +653,34 @@ Module Contents
built with :func:`datafusion.col`. On expressions are used to support
in-equality predicates. Equality predicates are correctly optimized.
- Example::
+ Use :py:meth:`DataFrame.col` on each DataFrame **before** the join to
+ obtain fully qualified column references. These qualified references
+ can then be used in the join predicate and to disambiguate columns
+ with the same name when selecting from the result.
- from datafusion import col
- df.join_on(other_df, col("id") == col("other_id"))
+ .. rubric:: Examples
+
+ Join with unique column names:
+
+ >>> ctx = dfn.SessionContext()
+ >>> left = ctx.from_pydict({"a": [1, 2], "x": ["a", "b"]})
+ >>> right = ctx.from_pydict({"b": [1, 2], "y": ["c", "d"]})
+ >>> left.join_on(
+ ... right, col("a") == col("b")
+ ... ).sort(col("x")).to_pydict()
+ {'a': [1, 2], 'x': ['a', 'b'], 'b': [1, 2], 'y': ['c', 'd']}
+
+ Use :py:meth:`col` to disambiguate shared column names:
+
+ >>> left = ctx.from_pydict({"id": [1, 2], "val": [10, 20]})
+ >>> right = ctx.from_pydict({"id": [1, 2], "val": [30, 40]})
+ >>> joined = left.join_on(
+ ... right, left.col("id") == right.col("id"), how="inner"
+ ... )
+ >>> joined.select(
+ ... left.col("id"), left.col("val"), right.col("val").alias("rval")
+ ... ).sort(left.col("id")).to_pydict()
+ {'id': [1, 2], 'val': [10, 20], 'rval': [30, 40]}
:param right: Other DataFrame to join with.
:param on_exprs: single or multiple (in)-equality predicates.
@@ -511,14 +724,13 @@ Module Contents
The expression is created and processed against the current schema.
- Example::
-
- from datafusion import col, lit
- df.parse_sql_expr("a > 1")
-
- should produce:
+ .. rubric:: Examples
- col("a") > lit(1)
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> expr = df.parse_sql_expr("a > 1")
+ >>> df.filter(expr).to_pydict()
+ {'a': [2, 3]}
:param expr: Expression string to be converted to datafusion expression
@@ -581,14 +793,6 @@ Module Contents
- .. py:method:: select_columns(*args: str) -> DataFrame
-
- Filter the DataFrame by columns.
-
- :returns: DataFrame only containing the specified columns.
-
-
-
.. py:method:: select_exprs(*args: str) -> DataFrame
Project arbitrary list of expression strings into a new DataFrame.
@@ -621,6 +825,29 @@ Module Contents
+ .. py:method:: sort_by(*exprs: datafusion.expr.Expr | str) -> DataFrame
+
+ Sort the DataFrame by column expressions in ascending order.
+
+ This is a convenience method that sorts the DataFrame by the given
+ expressions in ascending order with nulls last. For more control over
+ sort direction and null ordering, use :py:meth:`sort` instead.
+
+ :param exprs: Expressions or column names to sort by.
+
+ :returns: DataFrame after sorting.
+
+ .. rubric:: Examples
+
+ Sort by a single column:
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [3, 1, 2]})
+ >>> df.sort_by("a").to_pydict()
+ {'a': [1, 2, 3]}
+
+
+
.. py:method:: tail(n: int = 5) -> DataFrame
Return a new :py:class:`DataFrame` with a limited number of rows.
@@ -678,15 +905,18 @@ Module Contents
Apply a function to the current DataFrame which returns another DataFrame.
- This is useful for chaining together multiple functions. For example::
-
- def add_3(df: DataFrame) -> DataFrame:
- return df.with_column("modified", lit(3))
+ This is useful for chaining together multiple functions.
- def within_limit(df: DataFrame, limit: int) -> DataFrame:
- return df.filter(col("a") < lit(limit)).distinct()
+ .. rubric:: Examples
- df = df.transform(modify_df).transform(within_limit, 4)
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> def add_3(df):
+ ... return df.with_column("modified", dfn.lit(3))
+ >>> def within_limit(df: DataFrame, limit: int) -> DataFrame:
+ ... return df.filter(col("a") < lit(limit)).distinct()
+ >>> df.transform(add_3).transform(within_limit, 4).sort("a").to_pydict()
+ {'a': [1, 2, 3], 'modified': [3, 3, 3]}
:param func: A callable function that takes a DataFrame as it's first argument
:param args: Zero or more arguments to pass to `func`
@@ -709,29 +939,102 @@ Module Contents
- .. py:method:: union_distinct(other: DataFrame) -> DataFrame
+ .. py:method:: union_by_name(other: DataFrame, distinct: bool = False) -> DataFrame
- Calculate the distinct union of two :py:class:`DataFrame`.
+ Union two :py:class:`DataFrame` matching columns by name.
- The two :py:class:`DataFrame` must have exactly the same schema.
- Any duplicate rows are discarded.
+ Unlike :py:meth:`union` which matches columns by position, this method
+ matches columns by their names, allowing DataFrames with different
+ column orders to be combined.
:param other: DataFrame to union with.
+ :param distinct: If ``True``, duplicate rows are removed from the result.
- :returns: DataFrame after union.
+ :returns: DataFrame after union by name.
+
+ .. rubric:: Examples
+ Combine DataFrames with different column orders:
+ >>> ctx = dfn.SessionContext()
+ >>> df1 = ctx.from_pydict({"a": [1], "b": [10]})
+ >>> df2 = ctx.from_pydict({"b": [20], "a": [2]})
+ >>> df1.union_by_name(df2).sort("a").to_pydict()
+ {'a': [1, 2], 'b': [10, 20]}
- .. py:method:: unnest_columns(*columns: str, preserve_nulls: bool = True) -> DataFrame
+ Union by name with deduplication:
+
+ >>> df1 = ctx.from_pydict({"a": [1, 1], "b": [10, 10]})
+ >>> df2 = ctx.from_pydict({"b": [10], "a": [1]})
+ >>> df1.union_by_name(df2, distinct=True).to_pydict()
+ {'a': [1], 'b': [10]}
+
+
+
+ .. py:method:: union_distinct(other: DataFrame) -> DataFrame
+
+ Calculate the distinct union of two :py:class:`DataFrame`.
+
+ .. seealso:: :py:meth:`union`
+
+
+
+ .. py:method:: unnest_columns(*columns: str, preserve_nulls: bool = True, recursions: list[tuple[str, str, int]] | None = None) -> DataFrame
Expand columns of arrays into a single row per array element.
:param columns: Column names to perform unnest operation on.
:param preserve_nulls: If False, rows with null entries will not be
returned.
+ :param recursions: Optional list of ``(input_column, output_column, depth)``
+ tuples that control how deeply nested columns are unnested. Any
+ column not mentioned here is unnested with depth 1.
:returns: A DataFrame with the columns expanded.
+ .. rubric:: Examples
+
+ Unnest an array column:
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2], [3]], "b": ["x", "y"]})
+ >>> df.unnest_columns("a").to_pydict()
+ {'a': [1, 2, 3], 'b': ['x', 'x', 'y']}
+
+ With explicit recursion depth:
+
+ >>> df.unnest_columns("a", recursions=[("a", "a", 1)]).to_pydict()
+ {'a': [1, 2, 3], 'b': ['x', 'x', 'y']}
+
+
+
+ .. py:method:: window(*exprs: datafusion.expr.Expr) -> DataFrame
+
+ Add window function columns to the DataFrame.
+
+ Applies the given window function expressions and appends the results
+ as new columns.
+
+ :param exprs: Window function expressions to evaluate.
+
+ :returns: DataFrame with new window function columns appended.
+
+ .. rubric:: Examples
+
+ Add a row number within each group:
+
+ >>> import datafusion.functions as f
+ >>> from datafusion import col
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3], "b": ["x", "x", "y"]})
+ >>> df = df.window(
+ ... f.row_number(
+ ... partition_by=[col("b")], order_by=[col("a")]
+ ... ).alias("rn")
+ ... )
+ >>> "rn" in df.schema().names
+ True
+
.. py:method:: with_column(name: str, expr: datafusion.expr.Expr | str) -> DataFrame
@@ -742,10 +1045,12 @@ Module Contents
:func:`datafusion.col` or :func:`datafusion.lit`, or a SQL expression
string that will be parsed against the DataFrame schema.
- Example::
+ .. rubric:: Examples
- from datafusion import col, lit
- df.with_column("b", col("a") + lit(1))
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2]})
+ >>> df.with_column("b", col("a") + lit(10)).to_pydict()
+ {'a': [1, 2], 'b': [11, 12]}
:param name: Name of the column to add.
:param expr: Expression to compute the column.
@@ -891,6 +1196,44 @@ Module Contents
.. py:attribute:: _raw_write_options
+.. py:class:: ExplainFormat
+
+ Bases: :py:obj:`enum.Enum`
+
+
+ Output format for explain plans.
+
+ Controls how the query plan is rendered in :py:meth:`DataFrame.explain`.
+
+
+ .. py:attribute:: GRAPHVIZ
+ :value: 'graphviz'
+
+
+ Graphviz DOT format for graph rendering.
+
+
+ .. py:attribute:: INDENT
+ :value: 'indent'
+
+
+ Default indented text format.
+
+
+ .. py:attribute:: PGJSON
+ :value: 'pgjson'
+
+
+ PostgreSQL-compatible JSON format for use with visualization tools.
+
+
+ .. py:attribute:: TREE
+ :value: 'tree'
+
+
+ Tree-style visual format with box-drawing characters.
+
+
.. py:class:: InsertOp
Bases: :py:obj:`enum.Enum`
diff --git a/_sources/autoapi/datafusion/dataframe_formatter/index.rst.txt b/_sources/autoapi/datafusion/dataframe_formatter/index.rst.txt
index 243627a9..7d0f406a 100644
--- a/_sources/autoapi/datafusion/dataframe_formatter/index.rst.txt
+++ b/_sources/autoapi/datafusion/dataframe_formatter/index.rst.txt
@@ -476,7 +476,7 @@ Module Contents
.. rubric:: Example
- >>> from datafusion.html_formatter import configure_formatter
+ >>> from datafusion.dataframe_formatter import configure_formatter
>>> configure_formatter(
... max_cell_length=50,
... max_height=500,
@@ -497,7 +497,7 @@ Module Contents
.. rubric:: Example
- >>> from datafusion.html_formatter import get_formatter
+ >>> from datafusion.dataframe_formatter import get_formatter
>>> formatter = get_formatter()
>>> formatter.max_cell_length = 50 # Increase cell length
@@ -511,7 +511,7 @@ Module Contents
.. rubric:: Example
- >>> from datafusion.html_formatter import reset_formatter
+ >>> from datafusion.dataframe_formatter import reset_formatter
>>> reset_formatter() # Reset formatter to default settings
@@ -523,7 +523,7 @@ Module Contents
.. rubric:: Example
- >>> from datafusion.html_formatter import get_formatter, set_formatter
+ >>> from datafusion.dataframe_formatter import get_formatter, set_formatter
>>> custom_formatter = DataFrameHtmlFormatter(max_cell_length=100)
>>> set_formatter(custom_formatter)
diff --git a/_sources/autoapi/datafusion/expr/index.rst.txt b/_sources/autoapi/datafusion/expr/index.rst.txt
index 2bea04f4..47ea5a77 100644
--- a/_sources/autoapi/datafusion/expr/index.rst.txt
+++ b/_sources/autoapi/datafusion/expr/index.rst.txt
@@ -50,7 +50,6 @@ Attributes
datafusion.expr.Extension
datafusion.expr.FileType
datafusion.expr.Filter
- datafusion.expr.GroupingSet
datafusion.expr.ILike
datafusion.expr.InList
datafusion.expr.InSubquery
@@ -106,6 +105,7 @@ Classes
datafusion.expr.CaseBuilder
datafusion.expr.Expr
+ datafusion.expr.GroupingSet
datafusion.expr.SortExpr
datafusion.expr.Window
datafusion.expr.WindowFrame
@@ -128,16 +128,19 @@ Module Contents
Builder class for constructing case statements.
- An example usage would be as follows::
+ .. rubric:: Examples
- import datafusion.functions as f
- from datafusion import lit, col
- df.select(
- f.case(col("column_a"))
- .when(lit(1), lit("One"))
- .when(lit(2), lit("Two"))
- .otherwise(lit("Unknown"))
- )
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> result = df.select(
+ ... dfn.functions.case(dfn.col("a"))
+ ... .when(dfn.lit(1), dfn.lit("One"))
+ ... .when(dfn.lit(2), dfn.lit("Two"))
+ ... .otherwise(dfn.lit("Other"))
+ ... .alias("label")
+ ... )
+ >>> result.to_pydict()
+ {'label': ['One', 'Two', 'Other']}
Constructs a case builder.
@@ -536,14 +539,6 @@ Module Contents
- .. py:method:: display_name() -> str
-
- Returns the name of this expression as it should appear in a schema.
-
- This name will not include any CAST expressions.
-
-
-
.. py:method:: distinct() -> ExprFuncBuilder
Only evaluate distinct values for an aggregate function.
@@ -1014,6 +1009,138 @@ Module Contents
.. py:attribute:: expr
+.. py:class:: GroupingSet
+
+ Factory for creating grouping set expressions.
+
+ Grouping sets control how
+ :py:meth:`~datafusion.dataframe.DataFrame.aggregate` groups rows.
+ Instead of a single ``GROUP BY``, they produce multiple grouping
+ levels in one pass — subtotals, cross-tabulations, or arbitrary
+ column subsets.
+
+ Use :py:func:`~datafusion.functions.grouping` in the aggregate list
+ to tell which columns are aggregated across in each result row.
+
+
+ .. py:method:: cube(*exprs: Expr | str) -> Expr
+ :staticmethod:
+
+
+ Create a ``CUBE`` grouping set for use with ``aggregate()``.
+
+ ``CUBE`` generates all possible subsets of the given column list
+ as grouping sets. For example, ``cube(a, b)`` produces grouping
+ sets ``(a, b)``, ``(a)``, ``(b)``, and ``()`` (grand total).
+
+ This is equivalent to ``GROUP BY CUBE(a, b)`` in SQL.
+
+ :param \*exprs: Column expressions or column name strings to
+ include in the cube.
+
+ .. rubric:: Examples
+
+ With a single column, ``cube`` behaves identically to
+ :py:meth:`rollup`:
+
+ >>> from datafusion.expr import GroupingSet
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 1, 2], "b": [10, 20, 30]})
+ >>> result = df.aggregate(
+ ... [GroupingSet.cube(dfn.col("a"))],
+ ... [dfn.functions.sum(dfn.col("b")).alias("s"),
+ ... dfn.functions.grouping(dfn.col("a"))],
+ ... ).sort(dfn.col("a").sort(nulls_first=False))
+ >>> result.collect_column("s").to_pylist()
+ [30, 30, 60]
+
+ .. seealso::
+
+ :py:meth:`rollup`, :py:meth:`grouping_sets`,
+ :py:func:`~datafusion.functions.grouping`
+
+
+
+ .. py:method:: grouping_sets(*expr_lists: list[Expr | str]) -> Expr
+ :staticmethod:
+
+
+ Create explicit grouping sets for use with ``aggregate()``.
+
+ Each argument is a list of column expressions or column name
+ strings representing one grouping set. For example,
+ ``grouping_sets([a], [b])`` groups by ``a`` alone and by ``b``
+ alone in a single query.
+
+ This is equivalent to ``GROUP BY GROUPING SETS ((a), (b))`` in
+ SQL.
+
+ :param \*expr_lists: Each positional argument is a list of
+ expressions or column name strings forming one
+ grouping set.
+
+ .. rubric:: Examples
+
+ >>> from datafusion.expr import GroupingSet
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict(
+ ... {"a": ["x", "x", "y"], "b": ["m", "n", "m"],
+ ... "c": [1, 2, 3]})
+ >>> result = df.aggregate(
+ ... [GroupingSet.grouping_sets(
+ ... [dfn.col("a")], [dfn.col("b")])],
+ ... [dfn.functions.sum(dfn.col("c")).alias("s"),
+ ... dfn.functions.grouping(dfn.col("a")),
+ ... dfn.functions.grouping(dfn.col("b"))],
+ ... ).sort(
+ ... dfn.col("a").sort(nulls_first=False),
+ ... dfn.col("b").sort(nulls_first=False),
+ ... )
+ >>> result.collect_column("s").to_pylist()
+ [3, 3, 4, 2]
+
+ .. seealso::
+
+ :py:meth:`rollup`, :py:meth:`cube`,
+ :py:func:`~datafusion.functions.grouping`
+
+
+
+ .. py:method:: rollup(*exprs: Expr | str) -> Expr
+ :staticmethod:
+
+
+ Create a ``ROLLUP`` grouping set for use with ``aggregate()``.
+
+ ``ROLLUP`` generates all prefixes of the given column list as
+ grouping sets. For example, ``rollup(a, b)`` produces grouping
+ sets ``(a, b)``, ``(a)``, and ``()`` (grand total).
+
+ This is equivalent to ``GROUP BY ROLLUP(a, b)`` in SQL.
+
+ :param \*exprs: Column expressions or column name strings to
+ include in the rollup.
+
+ .. rubric:: Examples
+
+ >>> from datafusion.expr import GroupingSet
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 1, 2], "b": [10, 20, 30]})
+ >>> result = df.aggregate(
+ ... [GroupingSet.rollup(dfn.col("a"))],
+ ... [dfn.functions.sum(dfn.col("b")).alias("s"),
+ ... dfn.functions.grouping(dfn.col("a"))],
+ ... ).sort(dfn.col("a").sort(nulls_first=False))
+ >>> result.collect_column("s").to_pylist()
+ [30, 30, 60]
+
+ .. seealso::
+
+ :py:meth:`cube`, :py:meth:`grouping_sets`,
+ :py:func:`~datafusion.functions.grouping`
+
+
+
.. py:class:: SortExpr(expr: Expr, ascending: bool, nulls_first: bool)
Used to specify sorting on either a DataFrame or function.
@@ -1260,8 +1387,6 @@ Module Contents
.. py:data:: Filter
-.. py:data:: GroupingSet
-
.. py:data:: ILike
.. py:data:: InList
diff --git a/_sources/autoapi/datafusion/functions/index.rst.txt b/_sources/autoapi/datafusion/functions/index.rst.txt
index f1c50909..fac91311 100644
--- a/_sources/autoapi/datafusion/functions/index.rst.txt
+++ b/_sources/autoapi/datafusion/functions/index.rst.txt
@@ -9,6 +9,14 @@ datafusion.functions
+Attributes
+----------
+
+.. autoapisummary::
+
+ datafusion.functions.today
+
+
Functions
---------
@@ -24,10 +32,13 @@ Functions
datafusion.functions.approx_percentile_cont_with_weight
datafusion.functions.array
datafusion.functions.array_agg
+ datafusion.functions.array_any_value
datafusion.functions.array_append
datafusion.functions.array_cat
datafusion.functions.array_concat
+ datafusion.functions.array_contains
datafusion.functions.array_dims
+ datafusion.functions.array_distance
datafusion.functions.array_distinct
datafusion.functions.array_element
datafusion.functions.array_empty
@@ -40,6 +51,8 @@ Functions
datafusion.functions.array_intersect
datafusion.functions.array_join
datafusion.functions.array_length
+ datafusion.functions.array_max
+ datafusion.functions.array_min
datafusion.functions.array_ndims
datafusion.functions.array_pop_back
datafusion.functions.array_pop_front
@@ -56,11 +69,15 @@ Functions
datafusion.functions.array_replace_all
datafusion.functions.array_replace_n
datafusion.functions.array_resize
+ datafusion.functions.array_reverse
datafusion.functions.array_slice
datafusion.functions.array_sort
datafusion.functions.array_to_string
datafusion.functions.array_union
+ datafusion.functions.arrays_overlap
+ datafusion.functions.arrays_zip
datafusion.functions.arrow_cast
+ datafusion.functions.arrow_metadata
datafusion.functions.arrow_typeof
datafusion.functions.ascii
datafusion.functions.asin
@@ -87,6 +104,7 @@ Functions
datafusion.functions.col
datafusion.functions.concat
datafusion.functions.concat_ws
+ datafusion.functions.contains
datafusion.functions.corr
datafusion.functions.cos
datafusion.functions.cosh
@@ -99,7 +117,9 @@ Functions
datafusion.functions.cume_dist
datafusion.functions.current_date
datafusion.functions.current_time
+ datafusion.functions.current_timestamp
datafusion.functions.date_bin
+ datafusion.functions.date_format
datafusion.functions.date_part
datafusion.functions.date_trunc
datafusion.functions.datepart
@@ -108,6 +128,7 @@ Functions
datafusion.functions.degrees
datafusion.functions.dense_rank
datafusion.functions.digest
+ datafusion.functions.element_at
datafusion.functions.empty
datafusion.functions.encode
datafusion.functions.ends_with
@@ -120,6 +141,12 @@ Functions
datafusion.functions.floor
datafusion.functions.from_unixtime
datafusion.functions.gcd
+ datafusion.functions.gen_series
+ datafusion.functions.generate_series
+ datafusion.functions.get_field
+ datafusion.functions.greatest
+ datafusion.functions.grouping
+ datafusion.functions.ifnull
datafusion.functions.in_list
datafusion.functions.initcap
datafusion.functions.isnan
@@ -128,22 +155,35 @@ Functions
datafusion.functions.last_value
datafusion.functions.lcm
datafusion.functions.lead
+ datafusion.functions.least
datafusion.functions.left
datafusion.functions.length
datafusion.functions.levenshtein
+ datafusion.functions.list_any_value
datafusion.functions.list_append
datafusion.functions.list_cat
datafusion.functions.list_concat
+ datafusion.functions.list_contains
datafusion.functions.list_dims
+ datafusion.functions.list_distance
datafusion.functions.list_distinct
datafusion.functions.list_element
+ datafusion.functions.list_empty
datafusion.functions.list_except
datafusion.functions.list_extract
+ datafusion.functions.list_has
+ datafusion.functions.list_has_all
+ datafusion.functions.list_has_any
datafusion.functions.list_indexof
datafusion.functions.list_intersect
datafusion.functions.list_join
datafusion.functions.list_length
+ datafusion.functions.list_max
+ datafusion.functions.list_min
datafusion.functions.list_ndims
+ datafusion.functions.list_overlap
+ datafusion.functions.list_pop_back
+ datafusion.functions.list_pop_front
datafusion.functions.list_position
datafusion.functions.list_positions
datafusion.functions.list_prepend
@@ -157,10 +197,12 @@ Functions
datafusion.functions.list_replace_all
datafusion.functions.list_replace_n
datafusion.functions.list_resize
+ datafusion.functions.list_reverse
datafusion.functions.list_slice
datafusion.functions.list_sort
datafusion.functions.list_to_string
datafusion.functions.list_union
+ datafusion.functions.list_zip
datafusion.functions.ln
datafusion.functions.log
datafusion.functions.log10
@@ -171,6 +213,12 @@ Functions
datafusion.functions.make_array
datafusion.functions.make_date
datafusion.functions.make_list
+ datafusion.functions.make_map
+ datafusion.functions.make_time
+ datafusion.functions.map_entries
+ datafusion.functions.map_extract
+ datafusion.functions.map_keys
+ datafusion.functions.map_values
datafusion.functions.max
datafusion.functions.md5
datafusion.functions.mean
@@ -183,13 +231,16 @@ Functions
datafusion.functions.ntile
datafusion.functions.nullif
datafusion.functions.nvl
+ datafusion.functions.nvl2
datafusion.functions.octet_length
datafusion.functions.order_by
datafusion.functions.overlay
datafusion.functions.percent_rank
+ datafusion.functions.percentile_cont
datafusion.functions.pi
datafusion.functions.pow
datafusion.functions.power
+ datafusion.functions.quantile_cont
datafusion.functions.radians
datafusion.functions.random
datafusion.functions.range
@@ -213,6 +264,7 @@ Functions
datafusion.functions.reverse
datafusion.functions.right
datafusion.functions.round
+ datafusion.functions.row
datafusion.functions.row_number
datafusion.functions.rpad
datafusion.functions.rtrim
@@ -230,6 +282,8 @@ Functions
datafusion.functions.stddev_pop
datafusion.functions.stddev_samp
datafusion.functions.string_agg
+ datafusion.functions.string_to_array
+ datafusion.functions.string_to_list
datafusion.functions.strpos
datafusion.functions.struct
datafusion.functions.substr
@@ -238,7 +292,11 @@ Functions
datafusion.functions.sum
datafusion.functions.tan
datafusion.functions.tanh
+ datafusion.functions.to_char
+ datafusion.functions.to_date
datafusion.functions.to_hex
+ datafusion.functions.to_local_time
+ datafusion.functions.to_time
datafusion.functions.to_timestamp
datafusion.functions.to_timestamp_micros
datafusion.functions.to_timestamp_millis
@@ -248,14 +306,17 @@ Functions
datafusion.functions.translate
datafusion.functions.trim
datafusion.functions.trunc
+ datafusion.functions.union_extract
+ datafusion.functions.union_tag
datafusion.functions.upper
datafusion.functions.uuid
datafusion.functions.var
datafusion.functions.var_pop
+ datafusion.functions.var_population
datafusion.functions.var_samp
datafusion.functions.var_sample
+ datafusion.functions.version
datafusion.functions.when
- datafusion.functions.window
Module Contents
@@ -265,26 +326,40 @@ Module Contents
Return the absolute value of a given number.
- Returns:
- --------
- Expr
- A new expression representing the absolute value of the input expression.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [-1, 0, 1]})
+ >>> result = df.select(dfn.functions.abs(dfn.col("a")).alias("abs"))
+ >>> result.collect_column("abs")[0].as_py()
+ 1
.. py:function:: acos(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the arc cosine or inverse cosine of a number.
- Returns:
- --------
- Expr
- A new expression representing the arc cosine of the input expression.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0]})
+ >>> result = df.select(dfn.functions.acos(dfn.col("a")).alias("acos"))
+ >>> result.collect_column("acos")[0].as_py()
+ 0.0
.. py:function:: acosh(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns inverse hyperbolic cosine.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0]})
+ >>> result = df.select(dfn.functions.acosh(dfn.col("a")).alias("acosh"))
+ >>> result.collect_column("acosh")[0].as_py()
+ 0.0
+
.. py:function:: alias(expr: datafusion.expr.Expr, name: str, metadata: dict[str, str] | None = None) -> datafusion.expr.Expr
@@ -294,7 +369,27 @@ Module Contents
:param name: The alias name
:param metadata: Optional metadata to attach to the column
- :returns: An expression with the given alias
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2]})
+ >>> result = df.select(
+ ... dfn.functions.alias(
+ ... dfn.col("a"), "b"
+ ... )
+ ... )
+ >>> result.collect_column("b")[0].as_py()
+ 1
+
+ >>> result = df.select(
+ ... dfn.functions.alias(
+ ... dfn.col("a"), "b", metadata={"info": "test"}
+ ... )
+ ... )
+ >>> result.schema()
+ b: int64
+ -- field metadata --
+ info: 'test'
.. py:function:: approx_distinct(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -311,6 +406,25 @@ Module Contents
:param expression: Values to check for distinct entries
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 1, 2, 3]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.approx_distinct(
+ ... dfn.col("a")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py() == 3
+ True
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.approx_distinct(
+ ... dfn.col("a"),
+ ... filter=dfn.col("a") > dfn.lit(1)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py() == 2
+ True
+
.. py:function:: approx_median(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -325,6 +439,25 @@ Module Contents
:param expression: Values to find the median for
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.approx_median(
+ ... dfn.col("a")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2.0
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.approx_median(
+ ... dfn.col("a"),
+ ... filter=dfn.col("a") > dfn.lit(1.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2.5
+
.. py:function:: approx_percentile_cont(sort_expression: datafusion.expr.Expr | datafusion.expr.SortExpr, percentile: float, num_centroids: int | None = None, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -348,6 +481,26 @@ Module Contents
:param num_centroids: Max bin size for the t-digest algorithm
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0, 4.0, 5.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.approx_percentile_cont(
+ ... dfn.col("a"), 0.5
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 3.0
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.approx_percentile_cont(
+ ... dfn.col("a"), 0.5,
+ ... num_centroids=10,
+ ... filter=dfn.col("a") > dfn.lit(1.0),
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 3.5
+
.. py:function:: approx_percentile_cont_with_weight(sort_expression: datafusion.expr.Expr | datafusion.expr.SortExpr, weight: datafusion.expr.Expr, percentile: float, num_centroids: int | None = None, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -365,12 +518,32 @@ Module Contents
:param num_centroids: Max bin size for the t-digest algorithm
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0], "w": [1.0, 1.0, 1.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.approx_percentile_cont_with_weight(
+ ... dfn.col("a"), dfn.col("w"), 0.5
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2.0
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.approx_percentile_cont_with_weight(
+ ... dfn.col("a"), dfn.col("w"), 0.5,
+ ... num_centroids=10,
+ ... filter=dfn.col("a") > dfn.lit(1.0),
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2.5
+
.. py:function:: array(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns an array using the specified input expressions.
- This is an alias for :py:func:`make_array`.
+ .. seealso:: This is an alias for :py:func:`make_array`.
.. py:function:: array_agg(expression: datafusion.expr.Expr, distinct: bool = False, filter: datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None) -> datafusion.expr.Expr
@@ -389,64 +562,200 @@ Module Contents
:param filter: If provided, only compute against rows for which the filter is True
:param order_by: Order the resultant array values. Accepts column names or expressions.
- For example::
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.array_agg(
+ ... dfn.col("a")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ [1, 2, 3]
+
+ >>> df = ctx.from_pydict({"a": [3, 1, 2, 1]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.array_agg(
+ ... dfn.col("a"), distinct=True,
+ ... ).alias("v")])
+ >>> sorted(result.collect_column("v")[0].as_py())
+ [1, 2, 3]
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.array_agg(
+ ... dfn.col("a"),
+ ... filter=dfn.col("a") > dfn.lit(1),
+ ... order_by="a",
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ [2, 3]
- df.aggregate([], array_agg(col("a"), order_by="b"))
+
+.. py:function:: array_any_value(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the first non-null element in the array.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[None, 2, 3]]})
+ >>> result = df.select(
+ ... dfn.functions.array_any_value(dfn.col("a")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ 2
.. py:function:: array_append(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
Appends an element to the end of an array.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]]})
+ >>> result = df.select(
+ ... dfn.functions.array_append(dfn.col("a"), dfn.lit(4)).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [1, 2, 3, 4]
+
.. py:function:: array_cat(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
Concatenates the input arrays.
- This is an alias for :py:func:`array_concat`.
+ .. seealso:: This is an alias for :py:func:`array_concat`.
.. py:function:: array_concat(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
Concatenates the input arrays.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2]], "b": [[3, 4]]})
+ >>> result = df.select(
+ ... dfn.functions.array_concat(dfn.col("a"), dfn.col("b")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [1, 2, 3, 4]
+
+
+.. py:function:: array_contains(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns true if the element appears in the array, otherwise false.
+
+ .. seealso:: This is an alias for :py:func:`array_has`.
+
.. py:function:: array_dims(array: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns an array of the array's dimensions.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]]})
+ >>> result = df.select(dfn.functions.array_dims(dfn.col("a")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [3]
+
+
+.. py:function:: array_distance(array1: datafusion.expr.Expr, array2: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the Euclidean distance between two numeric arrays.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1.0, 2.0]], "b": [[1.0, 4.0]]})
+ >>> result = df.select(
+ ... dfn.functions.array_distance(
+ ... dfn.col("a"), dfn.col("b"),
+ ... ).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ 2.0
+
.. py:function:: array_distinct(array: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns distinct values from the array after removing duplicates.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 1, 2, 3]]})
+ >>> result = df.select(
+ ... dfn.functions.array_distinct(
+ ... dfn.col("a")
+ ... ).alias("result")
+ ... )
+ >>> sorted(
+ ... result.collect_column("result")[0].as_py()
+ ... )
+ [1, 2, 3]
+
.. py:function:: array_element(array: datafusion.expr.Expr, n: datafusion.expr.Expr) -> datafusion.expr.Expr
Extracts the element with the index n from the array.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[10, 20, 30]]})
+ >>> result = df.select(
+ ... dfn.functions.array_element(dfn.col("a"), dfn.lit(2)).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ 20
+
.. py:function:: array_empty(array: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns a boolean indicating whether the array is empty.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2]]})
+ >>> result = df.select(dfn.functions.array_empty(dfn.col("a")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ False
+
.. py:function:: array_except(array1: datafusion.expr.Expr, array2: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the elements that appear in ``array1`` but not in ``array2``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[2, 3, 4]]})
+ >>> result = df.select(
+ ... dfn.functions.array_except(dfn.col("a"), dfn.col("b")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [1]
+
.. py:function:: array_extract(array: datafusion.expr.Expr, n: datafusion.expr.Expr) -> datafusion.expr.Expr
Extracts the element with the index n from the array.
- This is an alias for :py:func:`array_element`.
+ .. seealso:: This is an alias for :py:func:`array_element`.
.. py:function:: array_has(first_array: datafusion.expr.Expr, second_array: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns true if the element appears in the first array, otherwise false.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]]})
+ >>> result = df.select(
+ ... dfn.functions.array_has(dfn.col("a"), dfn.lit(2)).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ True
+
.. py:function:: array_has_all(first_array: datafusion.expr.Expr, second_array: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -455,6 +764,15 @@ Module Contents
Returns true if each element of the second array appears in the first array.
Otherwise, it returns false.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[1, 2]]})
+ >>> result = df.select(
+ ... dfn.functions.array_has_all(dfn.col("a"), dfn.col("b")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ True
+
.. py:function:: array_has_any(first_array: datafusion.expr.Expr, second_array: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -463,104 +781,287 @@ Module Contents
Returns true if at least one element of the second array appears in the first
array. Otherwise, it returns false.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[2, 5]]})
+ >>> result = df.select(
+ ... dfn.functions.array_has_any(dfn.col("a"), dfn.col("b")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ True
+
.. py:function:: array_indexof(array: datafusion.expr.Expr, element: datafusion.expr.Expr, index: int | None = 1) -> datafusion.expr.Expr
Return the position of the first occurrence of ``element`` in ``array``.
- This is an alias for :py:func:`array_position`.
+ .. seealso:: This is an alias for :py:func:`array_position`.
.. py:function:: array_intersect(array1: datafusion.expr.Expr, array2: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the intersection of ``array1`` and ``array2``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[2, 3, 4]]})
+ >>> result = df.select(
+ ... dfn.functions.array_intersect(
+ ... dfn.col("a"), dfn.col("b")
+ ... ).alias("result")
+ ... )
+ >>> sorted(
+ ... result.collect_column("result")[0].as_py()
+ ... )
+ [2, 3]
+
.. py:function:: array_join(expr: datafusion.expr.Expr, delimiter: datafusion.expr.Expr) -> datafusion.expr.Expr
Converts each element to its text representation.
- This is an alias for :py:func:`array_to_string`.
+ .. seealso:: This is an alias for :py:func:`array_to_string`.
.. py:function:: array_length(array: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the length of the array.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]]})
+ >>> result = df.select(dfn.functions.array_length(dfn.col("a")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ 3
+
+
+.. py:function:: array_max(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the maximum value in the array.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]]})
+ >>> result = df.select(
+ ... dfn.functions.array_max(dfn.col("a")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ 3
+
+
+.. py:function:: array_min(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the minimum value in the array.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]]})
+ >>> result = df.select(
+ ... dfn.functions.array_min(dfn.col("a")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ 1
+
.. py:function:: array_ndims(array: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the number of dimensions of the array.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]]})
+ >>> result = df.select(dfn.functions.array_ndims(dfn.col("a")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ 1
+
.. py:function:: array_pop_back(array: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the array without the last element.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]]})
+ >>> result = df.select(
+ ... dfn.functions.array_pop_back(dfn.col("a")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [1, 2]
+
.. py:function:: array_pop_front(array: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the array without the first element.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]]})
+ >>> result = df.select(
+ ... dfn.functions.array_pop_front(dfn.col("a")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [2, 3]
+
.. py:function:: array_position(array: datafusion.expr.Expr, element: datafusion.expr.Expr, index: int | None = 1) -> datafusion.expr.Expr
Return the position of the first occurrence of ``element`` in ``array``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[10, 20, 30]]})
+ >>> result = df.select(
+ ... dfn.functions.array_position(
+ ... dfn.col("a"), dfn.lit(20)
+ ... ).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ 2
+
+ Use ``index`` to start searching from a given position:
+
+ >>> df = ctx.from_pydict({"a": [[10, 20, 10, 20]]})
+ >>> result = df.select(
+ ... dfn.functions.array_position(
+ ... dfn.col("a"), dfn.lit(20), index=3,
+ ... ).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ 4
+
.. py:function:: array_positions(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
Searches for an element in the array and returns all occurrences.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 1]]})
+ >>> result = df.select(
+ ... dfn.functions.array_positions(dfn.col("a"), dfn.lit(1)).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [1, 3]
+
.. py:function:: array_prepend(element: datafusion.expr.Expr, array: datafusion.expr.Expr) -> datafusion.expr.Expr
Prepends an element to the beginning of an array.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2]]})
+ >>> result = df.select(
+ ... dfn.functions.array_prepend(dfn.lit(0), dfn.col("a")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [0, 1, 2]
+
.. py:function:: array_push_back(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
Appends an element to the end of an array.
- This is an alias for :py:func:`array_append`.
+ .. seealso:: This is an alias for :py:func:`array_append`.
.. py:function:: array_push_front(element: datafusion.expr.Expr, array: datafusion.expr.Expr) -> datafusion.expr.Expr
Prepends an element to the beginning of an array.
- This is an alias for :py:func:`array_prepend`.
+ .. seealso:: This is an alias for :py:func:`array_prepend`.
.. py:function:: array_remove(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
Removes the first element from the array equal to the given value.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 1]]})
+ >>> result = df.select(
+ ... dfn.functions.array_remove(dfn.col("a"), dfn.lit(1)).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [2, 1]
+
.. py:function:: array_remove_all(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
Removes all elements from the array equal to the given value.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 1]]})
+ >>> result = df.select(
+ ... dfn.functions.array_remove_all(
+ ... dfn.col("a"), dfn.lit(1)
+ ... ).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [2]
+
.. py:function:: array_remove_n(array: datafusion.expr.Expr, element: datafusion.expr.Expr, max: datafusion.expr.Expr) -> datafusion.expr.Expr
Removes the first ``max`` elements from the array equal to the given value.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 1, 1]]})
+ >>> result = df.select(
+ ... dfn.functions.array_remove_n(dfn.col("a"), dfn.lit(1),
+ ... dfn.lit(2)).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [2, 1]
+
.. py:function:: array_repeat(element: datafusion.expr.Expr, count: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns an array containing ``element`` ``count`` times.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> result = df.select(
+ ... dfn.functions.array_repeat(dfn.lit(3), dfn.lit(3)).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [3, 3, 3]
+
.. py:function:: array_replace(array: datafusion.expr.Expr, from_val: datafusion.expr.Expr, to_val: datafusion.expr.Expr) -> datafusion.expr.Expr
Replaces the first occurrence of ``from_val`` with ``to_val``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 1]]})
+ >>> result = df.select(
+ ... dfn.functions.array_replace(dfn.col("a"), dfn.lit(1),
+ ... dfn.lit(9)).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [9, 2, 1]
+
.. py:function:: array_replace_all(array: datafusion.expr.Expr, from_val: datafusion.expr.Expr, to_val: datafusion.expr.Expr) -> datafusion.expr.Expr
Replaces all occurrences of ``from_val`` with ``to_val``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 1]]})
+ >>> result = df.select(
+ ... dfn.functions.array_replace_all(dfn.col("a"), dfn.lit(1),
+ ... dfn.lit(9)).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [9, 2, 9]
+
.. py:function:: array_replace_n(array: datafusion.expr.Expr, from_val: datafusion.expr.Expr, to_val: datafusion.expr.Expr, max: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -569,6 +1070,16 @@ Module Contents
Replaces the first ``max`` occurrences of the specified element with another
specified element.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 1, 1]]})
+ >>> result = df.select(
+ ... dfn.functions.array_replace_n(dfn.col("a"), dfn.lit(1), dfn.lit(9),
+ ... dfn.lit(2)).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [9, 2, 9, 1]
+
.. py:function:: array_resize(array: datafusion.expr.Expr, size: datafusion.expr.Expr, value: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -577,11 +1088,56 @@ Module Contents
If ``size`` is greater than the ``array`` length, the additional entries will
be filled with the given ``value``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2]]})
+ >>> result = df.select(
+ ... dfn.functions.array_resize(dfn.col("a"), dfn.lit(4),
+ ... dfn.lit(0)).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [1, 2, 0, 0]
+
+
+.. py:function:: array_reverse(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Reverses the order of elements in the array.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]]})
+ >>> result = df.select(
+ ... dfn.functions.array_reverse(dfn.col("a")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [3, 2, 1]
+
.. py:function:: array_slice(array: datafusion.expr.Expr, begin: datafusion.expr.Expr, end: datafusion.expr.Expr, stride: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Returns a slice of the array.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3, 4]]})
+ >>> result = df.select(
+ ... dfn.functions.array_slice(
+ ... dfn.col("a"), dfn.lit(2), dfn.lit(3)
+ ... ).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [2, 3]
+
+ Use ``stride`` to skip elements:
+
+ >>> result = df.select(
+ ... dfn.functions.array_slice(
+ ... dfn.col("a"), dfn.lit(1), dfn.lit(4),
+ ... stride=dfn.lit(2),
+ ... ).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [1, 3]
+
.. py:function:: array_sort(array: datafusion.expr.Expr, descending: bool = False, null_first: bool = False) -> datafusion.expr.Expr
@@ -591,11 +1147,39 @@ Module Contents
:param descending: If True, sorts in descending order.
:param null_first: If True, nulls will be returned at the beginning of the array.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[3, 1, 2]]})
+ >>> result = df.select(
+ ... dfn.functions.array_sort(
+ ... dfn.col("a")
+ ... ).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [1, 2, 3]
+
+ >>> df = ctx.from_pydict({"a": [[3, None, 1]]})
+ >>> result = df.select(
+ ... dfn.functions.array_sort(
+ ... dfn.col("a"), descending=True, null_first=True,
+ ... ).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [None, 3, 1]
+
.. py:function:: array_to_string(expr: datafusion.expr.Expr, delimiter: datafusion.expr.Expr) -> datafusion.expr.Expr
Converts each element to its text representation.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]]})
+ >>> result = df.select(
+ ... dfn.functions.array_to_string(dfn.col("a"), dfn.lit(",")).alias("s"))
+ >>> result.collect_column("s")[0].as_py()
+ '1,2,3'
+
.. py:function:: array_union(array1: datafusion.expr.Expr, array2: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -603,46 +1187,193 @@ Module Contents
Duplicate rows will not be returned.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[2, 3, 4]]})
+ >>> result = df.select(
+ ... dfn.functions.array_union(
+ ... dfn.col("a"), dfn.col("b")
+ ... ).alias("result")
+ ... )
+ >>> sorted(
+ ... result.collect_column("result")[0].as_py()
+ ... )
+ [1, 2, 3, 4]
+
+
+.. py:function:: arrays_overlap(first_array: datafusion.expr.Expr, second_array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns true if any element appears in both arrays.
+
+ .. seealso:: This is an alias for :py:func:`array_has_any`.
-.. py:function:: arrow_cast(expr: datafusion.expr.Expr, data_type: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+.. py:function:: arrays_zip(*arrays: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Combines multiple arrays into a single array of structs.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2]], "b": [[3, 4]]})
+ >>> result = df.select(
+ ... dfn.functions.arrays_zip(dfn.col("a"), dfn.col("b")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [{'c0': 1, 'c1': 3}, {'c0': 2, 'c1': 4}]
+
+
+.. py:function:: arrow_cast(expr: datafusion.expr.Expr, data_type: datafusion.expr.Expr | str | pyarrow.DataType) -> datafusion.expr.Expr
Casts an expression to a specified data type.
+ The ``data_type`` can be a string, a ``pyarrow.DataType``, or an
+ ``Expr``. For simple types, :py:meth:`Expr.cast()
+ <datafusion.expr.Expr.cast>` is more concise
+ (e.g., ``col("a").cast(pa.float64())``). Use ``arrow_cast`` when
+ you want to specify the target type as a string using DataFusion's
+ type syntax, which can be more readable for complex types like
+ ``"Timestamp(Nanosecond, None)"``.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> result = df.select(
+ ... dfn.functions.arrow_cast(dfn.col("a"), "Float64").alias("c")
+ ... )
+ >>> result.collect_column("c")[0].as_py()
+ 1.0
+
+ >>> result = df.select(
+ ... dfn.functions.arrow_cast(
+ ... dfn.col("a"), data_type=pa.float64()
+ ... ).alias("c")
+ ... )
+ >>> result.collect_column("c")[0].as_py()
+ 1.0
+
+
+.. py:function:: arrow_metadata(expr: datafusion.expr.Expr, key: datafusion.expr.Expr | str | None = None) -> datafusion.expr.Expr
+
+ Returns the metadata of the input expression.
+
+ If called with one argument, returns a Map of all metadata key-value pairs.
+ If called with two arguments, returns the value for the specified metadata key.
+
+ .. rubric:: Examples
+
+ >>> field = pa.field("val", pa.int64(), metadata={"k": "v"})
+ >>> schema = pa.schema([field])
+ >>> batch = pa.RecordBatch.from_arrays([pa.array([1])], schema=schema)
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.create_dataframe([[batch]])
+ >>> result = df.select(
+ ... dfn.functions.arrow_metadata(dfn.col("val")).alias("meta")
+ ... )
+ >>> ("k", "v") in result.collect_column("meta")[0].as_py()
+ True
+
+ >>> result = df.select(
+ ... dfn.functions.arrow_metadata(
+ ... dfn.col("val"), key="k"
+ ... ).alias("meta_val")
+ ... )
+ >>> result.collect_column("meta_val")[0].as_py()
+ 'v'
+
.. py:function:: arrow_typeof(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the Arrow type of the expression.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> result = df.select(dfn.functions.arrow_typeof(dfn.col("a")).alias("t"))
+ >>> result.collect_column("t")[0].as_py()
+ 'Int64'
+
.. py:function:: ascii(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the numeric code of the first character of the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["a","b","c"]})
+ >>> ascii_df = df.select(dfn.functions.ascii(dfn.col("a")).alias("ascii"))
+ >>> ascii_df.collect_column("ascii")[0].as_py()
+ 97
+
.. py:function:: asin(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the arc sine or inverse sine of a number.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0.0]})
+ >>> result = df.select(dfn.functions.asin(dfn.col("a")).alias("asin"))
+ >>> result.collect_column("asin")[0].as_py()
+ 0.0
+
.. py:function:: asinh(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns inverse hyperbolic sine.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0.0]})
+ >>> result = df.select(dfn.functions.asinh(dfn.col("a")).alias("asinh"))
+ >>> result.collect_column("asinh")[0].as_py()
+ 0.0
+
.. py:function:: atan(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns inverse tangent of a number.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0.0]})
+ >>> result = df.select(dfn.functions.atan(dfn.col("a")).alias("atan"))
+ >>> result.collect_column("atan")[0].as_py()
+ 0.0
+
.. py:function:: atan2(y: datafusion.expr.Expr, x: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns inverse tangent of a division given in the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"y": [0.0], "x": [1.0]})
+ >>> result = df.select(
+ ... dfn.functions.atan2(dfn.col("y"), dfn.col("x")).alias("atan2"))
+ >>> result.collect_column("atan2")[0].as_py()
+ 0.0
+
.. py:function:: atanh(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns inverse hyperbolic tangent.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0.0]})
+ >>> result = df.select(dfn.functions.atanh(dfn.col("a")).alias("atanh"))
+ >>> result.collect_column("atanh")[0].as_py()
+ 0.0
+
.. py:function:: avg(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -656,6 +1387,25 @@ Module Contents
:param expression: Values to combine into an array
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.avg(
+ ... dfn.col("a")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2.0
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.avg(
+ ... dfn.col("a"),
+ ... filter=dfn.col("a") > dfn.lit(1.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2.5
+
.. py:function:: bit_and(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -669,11 +1419,39 @@ Module Contents
:param expression: Argument to perform bitwise calculation on
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [7, 3]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.bit_and(
+ ... dfn.col("a")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 3
+
+ >>> df = ctx.from_pydict({"a": [7, 5, 3]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.bit_and(
+ ... dfn.col("a"),
+ ... filter=dfn.col("a") > dfn.lit(3)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 5
+
.. py:function:: bit_length(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the number of bits in the string argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["a","b","c"]})
+ >>> bit_df = df.select(dfn.functions.bit_length(dfn.col("a")).alias("bit_len"))
+ >>> bit_df.collect_column("bit_len")[0].as_py()
+ 8
+
.. py:function:: bit_or(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -687,6 +1465,28 @@ Module Contents
:param expression: Argument to perform bitwise calculation on
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.bit_or(
+ ... dfn.col("a")
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 3
+
+ >>> df = ctx.from_pydict({"a": [1, 2, 4]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.bit_or(
+ ... dfn.col("a"),
+ ... filter=dfn.col("a") > dfn.lit(1)
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 6
+
.. py:function:: bit_xor(expression: datafusion.expr.Expr, distinct: bool = False, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -701,6 +1501,28 @@ Module Contents
:param distinct: If True, evaluate each unique value of expression only once
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [5, 3]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.bit_xor(
+ ... dfn.col("a")
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 6
+
+ >>> df = ctx.from_pydict({"a": [5, 5, 3]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.bit_xor(
+ ... dfn.col("a"), distinct=True,
+ ... filter=dfn.col("a") > dfn.lit(3),
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 5
+
.. py:function:: bool_and(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -715,6 +1537,29 @@ Module Contents
:param expression: Argument to perform calculation on
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [True, True, False]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.bool_and(
+ ... dfn.col("a")
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ False
+
+ >>> df = ctx.from_pydict(
+ ... {"a": [True, True, False], "b": [1, 2, 3]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.bool_and(
+ ... dfn.col("a"),
+ ... filter=dfn.col("b") < dfn.lit(3)
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ True
+
.. py:function:: bool_or(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -729,16 +1574,55 @@ Module Contents
:param expression: Argument to perform calculation on
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [False, False, True]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.bool_or(
+ ... dfn.col("a")
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ True
+
+ >>> df = ctx.from_pydict(
+ ... {"a": [False, False, True], "b": [1, 2, 3]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.bool_or(
+ ... dfn.col("a"),
+ ... filter=dfn.col("b") < dfn.lit(3)
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ False
+
.. py:function:: btrim(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Removes all characters, spaces by default, from both sides of a string.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [" a "]})
+ >>> trim_df = df.select(dfn.functions.btrim(dfn.col("a")).alias("trimmed"))
+ >>> trim_df.collect_column("trimmed")[0].as_py()
+ 'a'
+
.. py:function:: cardinality(array: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the total number of elements in the array.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3]]})
+ >>> result = df.select(dfn.functions.cardinality(dfn.col("a")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ 3
+
.. py:function:: case(expr: datafusion.expr.Expr) -> datafusion.expr.CaseBuilder
@@ -748,41 +1632,108 @@ Module Contents
expression ``expr``. See :py:class:`~datafusion.expr.CaseBuilder` for
detailed usage.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> result = df.select(
+ ... dfn.functions.case(dfn.col("a")).when(dfn.lit(1),
+ ... dfn.lit("one")).otherwise(dfn.lit("other")).alias("c"))
+ >>> result.collect_column("c")[0].as_py()
+ 'one'
+
.. py:function:: cbrt(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the cube root of a number.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [27]})
+ >>> cbrt_df = df.select(dfn.functions.cbrt(dfn.col("a")).alias("cbrt"))
+ >>> cbrt_df.collect_column("cbrt")[0].as_py()
+ 3.0
+
.. py:function:: ceil(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the nearest integer greater than or equal to argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.9]})
+ >>> ceil_df = df.select(dfn.functions.ceil(dfn.col("a")).alias("ceil"))
+ >>> ceil_df.collect_column("ceil")[0].as_py()
+ 2.0
+
.. py:function:: char_length(string: datafusion.expr.Expr) -> datafusion.expr.Expr
The number of characters in the ``string``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(dfn.functions.char_length(dfn.col("a")).alias("len"))
+ >>> result.collect_column("len")[0].as_py()
+ 5
+
.. py:function:: character_length(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the number of characters in the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["abc","b","c"]})
+ >>> char_len_df = df.select(
+ ... dfn.functions.character_length(dfn.col("a")).alias("char_len"))
+ >>> char_len_df.collect_column("char_len")[0].as_py()
+ 3
+
.. py:function:: chr(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Converts the Unicode code point to a UTF8 character.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [65]})
+ >>> result = df.select(dfn.functions.chr(dfn.col("a")).alias("chr"))
+ >>> result.collect_column("chr")[0].as_py()
+ 'A'
+
.. py:function:: coalesce(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the value of the first expr in ``args`` which is not NULL.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [None, 1], "b": [2, 3]})
+ >>> result = df.select(
+ ... dfn.functions.coalesce(dfn.col("a"), dfn.col("b")).alias("c"))
+ >>> result.collect_column("c")[0].as_py()
+ 2
+
.. py:function:: col(name: str) -> datafusion.expr.Expr
Creates a column reference expression.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> df.select(dfn.functions.col("a")).collect_column("a")[0].as_py()
+ 1
+
.. py:function:: concat(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -790,6 +1741,16 @@ Module Contents
NULL arguments are ignored.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"], "b": [" world"]})
+ >>> result = df.select(
+ ... dfn.functions.concat(dfn.col("a"), dfn.col("b")).alias("c")
+ ... )
+ >>> result.collect_column("c")[0].as_py()
+ 'hello world'
+
.. py:function:: concat_ws(separator: str, *args: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -797,6 +1758,29 @@ Module Contents
``NULL`` arguments are ignored. ``separator`` should not be ``NULL``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"], "b": ["world"]})
+ >>> result = df.select(
+ ... dfn.functions.concat_ws("-", dfn.col("a"), dfn.col("b")).alias("c"))
+ >>> result.collect_column("c")[0].as_py()
+ 'hello-world'
+
+
+.. py:function:: contains(string: datafusion.expr.Expr, search_str: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns true if ``search_str`` is found within ``string`` (case-sensitive).
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["the quick brown fox"]})
+ >>> result = df.select(
+ ... dfn.functions.contains(dfn.col("a"), dfn.lit("brown")).alias("c"))
+ >>> result.collect_column("c")[0].as_py()
+ True
+
.. py:function:: corr(value_y: datafusion.expr.Expr, value_x: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -811,21 +1795,67 @@ Module Contents
:param value_x: The independent variable for correlation
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0], "b": [1.0, 2.0, 3.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.corr(
+ ... dfn.col("a"), dfn.col("b")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 1.0
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.corr(
+ ... dfn.col("a"), dfn.col("b"),
+ ... filter=dfn.col("a") > dfn.lit(1.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 1.0
+
.. py:function:: cos(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the cosine of the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0,-1,1]})
+ >>> cos_df = df.select(dfn.functions.cos(dfn.col("a")).alias("cos"))
+ >>> cos_df.collect_column("cos")[0].as_py()
+ 1.0
+
.. py:function:: cosh(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the hyperbolic cosine of the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0,-1,1]})
+ >>> cosh_df = df.select(dfn.functions.cosh(dfn.col("a")).alias("cosh"))
+ >>> cosh_df.collect_column("cosh")[0].as_py()
+ 1.0
+
.. py:function:: cot(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the cotangent of the argument.
+ .. rubric:: Examples
+
+ >>> from math import pi
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [pi / 4]})
+ >>> result = df.select(
+ ... dfn.functions.cot(dfn.col("a")).alias("cot")
+ ... )
+ >>> result.collect_column("cot")[0].as_py()
+ 1.0...
+
.. py:function:: count(expressions: datafusion.expr.Expr | list[datafusion.expr.Expr] | None = None, distinct: bool = False, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -840,6 +1870,26 @@ Module Contents
:param distinct: If True, a single entry for each distinct value will be in the result
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.count(
+ ... dfn.col("a")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 3
+
+ >>> df = ctx.from_pydict({"a": [1, 1, 2, 3]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.count(
+ ... dfn.col("a"), distinct=True,
+ ... filter=dfn.col("a") > dfn.lit(1),
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2
+
.. py:function:: count_star(filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -852,12 +1902,29 @@ Module Contents
:param filter: If provided, only count rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.count_star(
+ ... ).alias("cnt")])
+ >>> result.collect_column("cnt")[0].as_py()
+ 3
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.count_star(
+ ... filter=dfn.col("a") > dfn.lit(1)
+ ... ).alias("cnt")])
+ >>> result.collect_column("cnt")[0].as_py()
+ 2
+
.. py:function:: covar(value_y: datafusion.expr.Expr, value_x: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Computes the sample covariance.
- This is an alias for :py:func:`covar_samp`.
+ .. seealso:: This is an alias for :py:func:`covar_samp`.
.. py:function:: covar_pop(value_y: datafusion.expr.Expr, value_x: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -873,6 +1940,31 @@ Module Contents
:param value_x: The independent variable for covariance
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0, 5.0, 10.0], "b": [1.0, 2.0, 3.0]})
+ >>> result = df.aggregate(
+ ... [],
+ ... [dfn.functions.covar_pop(
+ ... dfn.col("a"), dfn.col("b")
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 3.0
+
+ >>> df = ctx.from_pydict(
+ ... {"a": [0.0, 1.0, 3.0], "b": [0.0, 1.0, 3.0]})
+ >>> result = df.aggregate(
+ ... [],
+ ... [dfn.functions.covar_pop(
+ ... dfn.col("a"), dfn.col("b"),
+ ... filter=dfn.col("a") > dfn.lit(0.0)
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 1.0
+
.. py:function:: covar_samp(value_y: datafusion.expr.Expr, value_x: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -887,6 +1979,25 @@ Module Contents
:param value_x: The independent variable for covariance
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0], "b": [4.0, 5.0, 6.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.covar_samp(
+ ... dfn.col("a"), dfn.col("b")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 1.0
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.covar_samp(
+ ... dfn.col("a"), dfn.col("b"),
+ ... filter=dfn.col("a") > dfn.lit(1.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 0.5
+
.. py:function:: cume_dist(partition_by: list[datafusion.expr.Expr] | datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None) -> datafusion.expr.Expr
@@ -910,59 +2021,173 @@ Module Contents
:param order_by: Set ordering within the window frame. Accepts
column names or expressions.
- For example::
-
- cume_dist(order_by="points")
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1., 2., 2., 3.]})
+ >>> result = df.select(
+ ... dfn.col("a"),
+ ... dfn.functions.cume_dist(
+ ... order_by="a"
+ ... ).alias("cd")
+ ... )
+ >>> result.collect_column("cd").to_pylist()
+ [0.25..., 0.75..., 0.75..., 1.0...]
+
+ >>> df = ctx.from_pydict(
+ ... {"g": ["a", "a", "b", "b"], "v": [1, 2, 3, 4]})
+ >>> result = df.select(
+ ... dfn.col("g"), dfn.col("v"),
+ ... dfn.functions.cume_dist(
+ ... partition_by=dfn.col("g"), order_by="v",
+ ... ).alias("cd"))
+ >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("cd").to_pylist()
+ [0.5, 1.0, 0.5, 1.0]
.. py:function:: current_date() -> datafusion.expr.Expr
Returns current UTC date as a Date32 value.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> result = df.select(
+ ... dfn.functions.current_date().alias("d")
+ ... )
+ >>> result.collect_column("d")[0].as_py() is not None
+ True
+
.. py:function:: current_time() -> datafusion.expr.Expr
Returns current UTC time as a Time64 value.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> result = df.select(
+ ... dfn.functions.current_time().alias("t")
+ ... )
+
+ Use .value instead of .as_py() because nanosecond timestamps
+ require pandas to convert to Python datetime objects.
+
+ >>> result.collect_column("t")[0].value > 0
+ True
+
+
+.. py:function:: current_timestamp() -> datafusion.expr.Expr
+
+ Returns the current timestamp in nanoseconds.
+
+ .. seealso:: This is an alias for :py:func:`now`.
+
.. py:function:: date_bin(stride: datafusion.expr.Expr, source: datafusion.expr.Expr, origin: datafusion.expr.Expr) -> datafusion.expr.Expr
Coerces an arbitrary timestamp to the start of the nearest specified interval.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"timestamp": ['2021-07-15 12:34:56', '2021-01-01']})
+ >>> result = df.select(
+ ... dfn.functions.date_bin(
+ ... dfn.string_literal("15 minutes"),
+ ... dfn.col("timestamp"),
+ ... dfn.string_literal("2001-01-01 00:00:00")
+ ... ).alias("b")
+ ... )
+ >>> str(result.collect_column("b")[0].as_py())
+ '2021-07-15 12:30:00'
+ >>> str(result.collect_column("b")[1].as_py())
+ '2021-01-01 00:00:00'
+
+
+.. py:function:: date_format(arg: datafusion.expr.Expr, formatter: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns a string representation of a date, time, timestamp or duration.
+
+ .. seealso:: This is an alias for :py:func:`to_char`.
+
.. py:function:: date_part(part: datafusion.expr.Expr, date: datafusion.expr.Expr) -> datafusion.expr.Expr
Extracts a subfield from the date.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["2021-07-15T00:00:00"]})
+ >>> df = df.select(dfn.functions.to_timestamp(dfn.col("a")).alias("a"))
+ >>> result = df.select(
+ ... dfn.functions.date_part(dfn.lit("year"), dfn.col("a")).alias("y"))
+ >>> result.collect_column("y")[0].as_py()
+ 2021
+
.. py:function:: date_trunc(part: datafusion.expr.Expr, date: datafusion.expr.Expr) -> datafusion.expr.Expr
Truncates the date to a specified level of precision.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["2021-07-15T12:34:56"]})
+ >>> df = df.select(dfn.functions.to_timestamp(dfn.col("a")).alias("a"))
+ >>> result = df.select(
+ ... dfn.functions.date_trunc(
+ ... dfn.lit("month"), dfn.col("a")
+ ... ).alias("t")
+ ... )
+ >>> str(result.collect_column("t")[0].as_py())
+ '2021-07-01 00:00:00'
+
.. py:function:: datepart(part: datafusion.expr.Expr, date: datafusion.expr.Expr) -> datafusion.expr.Expr
Return a specified part of a date.
- This is an alias for :py:func:`date_part`.
+ .. seealso:: This is an alias for :py:func:`date_part`.
.. py:function:: datetrunc(part: datafusion.expr.Expr, date: datafusion.expr.Expr) -> datafusion.expr.Expr
Truncates the date to a specified level of precision.
- This is an alias for :py:func:`date_trunc`.
+ .. seealso:: This is an alias for :py:func:`date_trunc`.
.. py:function:: decode(expr: datafusion.expr.Expr, encoding: datafusion.expr.Expr) -> datafusion.expr.Expr
Decode the ``input``, using the ``encoding``. encoding can be base64 or hex.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["aGVsbG8="]})
+ >>> result = df.select(
+ ... dfn.functions.decode(dfn.col("a"), dfn.lit("base64")).alias("dec"))
+ >>> result.collect_column("dec")[0].as_py()
+ b'hello'
+
.. py:function:: degrees(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Converts the argument from radians to degrees.
+ .. rubric:: Examples
+
+ >>> from math import pi
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0,pi,2*pi]})
+ >>> deg_df = df.select(dfn.functions.degrees(dfn.col("a")).alias("deg"))
+ >>> deg_df.collect_column("deg")[2].as_py()
+ 360.0
+
.. py:function:: dense_rank(partition_by: list[datafusion.expr.Expr] | datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None) -> datafusion.expr.Expr
@@ -985,9 +2210,27 @@ Module Contents
:param order_by: Set ordering within the window frame. Accepts
column names or expressions.
- For example::
-
- dense_rank(order_by="points")
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [10, 10, 20]})
+ >>> result = df.select(
+ ... dfn.col("a"),
+ ... dfn.functions.dense_rank(
+ ... order_by="a"
+ ... ).alias("dr"))
+ >>> result.sort(dfn.col("a")).collect_column("dr").to_pylist()
+ [1, 1, 2]
+
+ >>> df = ctx.from_pydict(
+ ... {"g": ["a", "a", "b", "b"], "v": [1, 1, 2, 3]})
+ >>> result = df.select(
+ ... dfn.col("g"), dfn.col("v"),
+ ... dfn.functions.dense_rank(
+ ... partition_by=dfn.col("g"), order_by="v",
+ ... ).alias("dr"))
+ >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("dr").to_pylist()
+ [1, 1, 1, 2]
.. py:function:: digest(value: datafusion.expr.Expr, method: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -997,38 +2240,94 @@ Module Contents
Standard algorithms are md5, sha224, sha256, sha384, sha512, blake2s,
blake2b, and blake3.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(
+ ... dfn.functions.digest(dfn.col("a"), dfn.lit("md5")).alias("d"))
+ >>> len(result.collect_column("d")[0].as_py()) > 0
+ True
+
+
+.. py:function:: element_at(map: datafusion.expr.Expr, key: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the value for a given key in the map.
+
+ Returns ``[None]`` if the key is absent.
+
+ .. seealso:: This is an alias for :py:func:`map_extract`.
+
.. py:function:: empty(array: datafusion.expr.Expr) -> datafusion.expr.Expr
- This is an alias for :py:func:`array_empty`.
+ Returns true if the array is empty.
+
+ .. seealso:: This is an alias for :py:func:`array_empty`.
.. py:function:: encode(expr: datafusion.expr.Expr, encoding: datafusion.expr.Expr) -> datafusion.expr.Expr
Encode the ``input``, using the ``encoding``. encoding can be base64 or hex.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(
+ ... dfn.functions.encode(dfn.col("a"), dfn.lit("base64")).alias("enc"))
+ >>> result.collect_column("enc")[0].as_py()
+ 'aGVsbG8'
+
.. py:function:: ends_with(arg: datafusion.expr.Expr, suffix: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns true if the ``string`` ends with the ``suffix``, false otherwise.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["abc","b","c"]})
+ >>> ends_with_df = df.select(
+ ... dfn.functions.ends_with(dfn.col("a"), dfn.lit("c")).alias("ends_with"))
+ >>> ends_with_df.collect_column("ends_with")[0].as_py()
+ True
+
.. py:function:: exp(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the exponential of the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0.0]})
+ >>> result = df.select(dfn.functions.exp(dfn.col("a")).alias("exp"))
+ >>> result.collect_column("exp")[0].as_py()
+ 1.0
+
.. py:function:: extract(part: datafusion.expr.Expr, date: datafusion.expr.Expr) -> datafusion.expr.Expr
Extracts a subfield from the date.
- This is an alias for :py:func:`date_part`.
+ .. seealso:: This is an alias for :py:func:`date_part`.
.. py:function:: factorial(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the factorial of the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [3]})
+ >>> result = df.select(
+ ... dfn.functions.factorial(dfn.col("a")).alias("factorial")
+ ... )
+ >>> result.collect_column("factorial")[0].as_py()
+ 6
+
.. py:function:: find_in_set(string: datafusion.expr.Expr, string_list: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -1039,6 +2338,15 @@ Module Contents
The string list is a string composed of substrings separated by ``,`` characters.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["b"]})
+ >>> result = df.select(
+ ... dfn.functions.find_in_set(dfn.col("a"), dfn.lit("a,b,c")).alias("pos"))
+ >>> result.collect_column("pos")[0].as_py()
+ 2
+
.. py:function:: first_value(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None, null_treatment: datafusion.common.NullTreatment = NullTreatment.RESPECT_NULLS) -> datafusion.expr.Expr
@@ -1055,53 +2363,302 @@ Module Contents
column names or expressions.
:param null_treatment: Assign whether to respect or ignore null values.
- For example::
-
- df.aggregate([], first_value(col("a"), order_by="ts"))
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [10, 20, 30]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.first_value(
+ ... dfn.col("a")
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 10
+
+ >>> df = ctx.from_pydict({"a": [None, 20, 10]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.first_value(
+ ... dfn.col("a"),
+ ... filter=dfn.col("a") > dfn.lit(10),
+ ... order_by="a",
+ ... null_treatment=dfn.common.NullTreatment.IGNORE_NULLS,
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 20
.. py:function:: flatten(array: datafusion.expr.Expr) -> datafusion.expr.Expr
Flattens an array of arrays into a single array.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[[1, 2], [3, 4]]]})
+ >>> result = df.select(dfn.functions.flatten(dfn.col("a")).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [1, 2, 3, 4]
+
.. py:function:: floor(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the nearest integer less than or equal to the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.9]})
+ >>> floor_df = df.select(dfn.functions.floor(dfn.col("a")).alias("floor"))
+ >>> floor_df.collect_column("floor")[0].as_py()
+ 1.0
+
.. py:function:: from_unixtime(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Converts an integer to RFC3339 timestamp format string.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0]})
+ >>> result = df.select(
+ ... dfn.functions.from_unixtime(
+ ... dfn.col("a")
+ ... ).alias("ts")
+ ... )
+ >>> str(result.collect_column("ts")[0].as_py())
+ '1970-01-01 00:00:00'
+
.. py:function:: gcd(x: datafusion.expr.Expr, y: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the greatest common divisor.
+ .. rubric:: Examples
-.. py:function:: in_list(arg: datafusion.expr.Expr, values: list[datafusion.expr.Expr], negated: bool = False) -> datafusion.expr.Expr
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [12], "b": [8]})
+ >>> result = df.select(
+ ... dfn.functions.gcd(dfn.col("a"), dfn.col("b")).alias("gcd")
+ ... )
+ >>> result.collect_column("gcd")[0].as_py()
+ 4
- Returns whether the argument is contained within the list ``values``.
+.. py:function:: gen_series(start: datafusion.expr.Expr, stop: datafusion.expr.Expr, step: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
-.. py:function:: initcap(string: datafusion.expr.Expr) -> datafusion.expr.Expr
+ Creates a list of values in the range between start and stop.
+
+ Unlike :py:func:`range`, this includes the upper bound.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0]})
+ >>> result = df.select(
+ ... dfn.functions.gen_series(
+ ... dfn.lit(1), dfn.lit(5),
+ ... ).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [1, 2, 3, 4, 5]
+
+ Specify a custom ``step``:
+
+ >>> result = df.select(
+ ... dfn.functions.gen_series(
+ ... dfn.lit(1), dfn.lit(10), step=dfn.lit(3),
+ ... ).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [1, 4, 7, 10]
+
+
+.. py:function:: generate_series(start: datafusion.expr.Expr, stop: datafusion.expr.Expr, step: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
+
+ Creates a list of values in the range between start and stop.
+
+ Unlike :py:func:`range`, this includes the upper bound.
+
+ .. seealso:: This is an alias for :py:func:`gen_series`.
+
+
+.. py:function:: get_field(expr: datafusion.expr.Expr, name: datafusion.expr.Expr | str) -> datafusion.expr.Expr
+
+ Extracts a field from a struct or map by name.
+
+ When the field name is a static string, the bracket operator
+ ``expr["field"]`` is a convenient shorthand. Use ``get_field``
+ when the field name is a dynamic expression.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1], "b": [2]})
+ >>> df = df.with_column(
+ ... "s",
+ ... dfn.functions.named_struct(
+ ... [("x", dfn.col("a")), ("y", dfn.col("b"))]
+ ... ),
+ ... )
+ >>> result = df.select(
+ ... dfn.functions.get_field(dfn.col("s"), "x").alias("x_val")
+ ... )
+ >>> result.collect_column("x_val")[0].as_py()
+ 1
+
+ Equivalent using bracket syntax:
+
+ >>> result = df.select(
+ ... dfn.col("s")["x"].alias("x_val")
+ ... )
+ >>> result.collect_column("x_val")[0].as_py()
+ 1
+
+
+.. py:function:: greatest(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the greatest value from a list of expressions.
+
+ Returns NULL if all expressions are NULL.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 3], "b": [2, 1]})
+ >>> result = df.select(
+ ... dfn.functions.greatest(dfn.col("a"), dfn.col("b")).alias("greatest"))
+ >>> result.collect_column("greatest")[0].as_py()
+ 2
+ >>> result.collect_column("greatest")[1].as_py()
+ 3
+
+
+.. py:function:: grouping(expression: datafusion.expr.Expr, distinct: bool = False, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
+
+ Indicates whether a column is aggregated across in the current row.
+
+ Returns 0 when the column is part of the grouping key for that row
+ (i.e., the row contains per-group results for that column). Returns 1
+ when the column is *not* part of the grouping key (i.e., the row's
+ aggregate spans all values of that column).
+
+ This function is meaningful with
+ :py:meth:`GroupingSet.rollup <datafusion.expr.GroupingSet.rollup>`,
+ :py:meth:`GroupingSet.cube <datafusion.expr.GroupingSet.cube>`, or
+ :py:meth:`GroupingSet.grouping_sets <datafusion.expr.GroupingSet.grouping_sets>`,
+ where different rows are grouped by different subsets of columns. In a
+ default aggregation without grouping sets every column is always part
+ of the key, so ``grouping()`` always returns 0.
+
+ .. warning::
+
+ Due to an upstream DataFusion limitation
+ (`#21411 <https://github.com/apache/datafusion/issues/21411>`_),
+ ``.alias()`` cannot be applied directly to a ``grouping()``
+ expression. Doing so will raise an error at execution time. To
+ rename the column, use
+ :py:meth:`~datafusion.dataframe.DataFrame.with_column_renamed`
+ on the result DataFrame instead.
+
+ :param expression: The column to check grouping status for
+ :param distinct: If True, compute on distinct values only
+ :param filter: If provided, only compute against rows for which the filter is True
+
+ .. rubric:: Examples
+
+ With :py:meth:`~datafusion.expr.GroupingSet.rollup`, the result
+ includes both per-group rows (``grouping(a) = 0``) and a
+ grand-total row where ``a`` is aggregated across
+ (``grouping(a) = 1``):
+
+ >>> from datafusion.expr import GroupingSet
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 1, 2], "b": [10, 20, 30]})
+ >>> result = df.aggregate(
+ ... [GroupingSet.rollup(dfn.col("a"))],
+ ... [dfn.functions.sum(dfn.col("b")).alias("s"),
+ ... dfn.functions.grouping(dfn.col("a"))],
+ ... ).sort(dfn.col("a").sort(nulls_first=False))
+ >>> result.collect_column("s").to_pylist()
+ [30, 30, 60]
+
+ .. seealso:: :py:class:`~datafusion.expr.GroupingSet`
+
+
+.. py:function:: ifnull(x: datafusion.expr.Expr, y: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns ``x`` if ``x`` is not NULL. Otherwise returns ``y``.
+
+ .. seealso:: This is an alias for :py:func:`nvl`.
+
+
+.. py:function:: in_list(arg: datafusion.expr.Expr, values: list[datafusion.expr.Expr], negated: bool = False) -> datafusion.expr.Expr
+
+ Returns whether the argument is contained within the list ``values``.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> result = df.select(
+ ... dfn.functions.in_list(
+ ... dfn.col("a"), [dfn.lit(1), dfn.lit(3)]
+ ... ).alias("in")
+ ... )
+ >>> result.collect_column("in").to_pylist()
+ [True, False, True]
+
+ >>> result = df.select(
+ ... dfn.functions.in_list(
+ ... dfn.col("a"), [dfn.lit(1), dfn.lit(3)],
+ ... negated=True,
+ ... ).alias("not_in")
+ ... )
+ >>> result.collect_column("not_in").to_pylist()
+ [False, True, False]
+
+
+.. py:function:: initcap(string: datafusion.expr.Expr) -> datafusion.expr.Expr
Set the initial letter of each word to capital.
Converts the first letter of each word in ``string`` to uppercase and the remaining
characters to lowercase.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["the cat"]})
+ >>> cap_df = df.select(dfn.functions.initcap(dfn.col("a")).alias("cap"))
+ >>> cap_df.collect_column("cap")[0].as_py()
+ 'The Cat'
+
.. py:function:: isnan(expr: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns true if a given number is +NaN or -NaN otherwise returns false.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0, np.nan]})
+ >>> result = df.select(dfn.functions.isnan(dfn.col("a")).alias("isnan"))
+ >>> result.collect_column("isnan")[1].as_py()
+ True
+
.. py:function:: iszero(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns true if a given number is +0.0 or -0.0 otherwise returns false.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0.0, 1.0]})
+ >>> result = df.select(dfn.functions.iszero(dfn.col("a")).alias("iz"))
+ >>> result.collect_column("iz")[0].as_py()
+ True
+
.. py:function:: lag(arg: datafusion.expr.Expr, shift_offset: int = 1, default_value: Any | None = None, partition_by: list[datafusion.expr.Expr] | datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None) -> datafusion.expr.Expr
@@ -1131,9 +2688,28 @@ Module Contents
:param order_by: Set ordering within the window frame. Accepts
column names or expressions.
- For example::
-
- lag(col("b"), order_by="ts")
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> result = df.select(
+ ... dfn.col("a"),
+ ... dfn.functions.lag(
+ ... dfn.col("a"), shift_offset=1,
+ ... default_value=0, order_by="a"
+ ... ).alias("lag"))
+ >>> result.sort(dfn.col("a")).collect_column("lag").to_pylist()
+ [0, 1, 2]
+
+ >>> df = ctx.from_pydict({"g": ["a", "a", "b"], "v": [1, 2, 3]})
+ >>> result = df.select(
+ ... dfn.col("g"), dfn.col("v"),
+ ... dfn.functions.lag(
+ ... dfn.col("v"), shift_offset=1, default_value=0,
+ ... partition_by=dfn.col("g"), order_by="v",
+ ... ).alias("lag"))
+ >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("lag").to_pylist()
+ [0, 1, 0]
.. py:function:: last_value(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None, null_treatment: datafusion.common.NullTreatment = NullTreatment.RESPECT_NULLS) -> datafusion.expr.Expr
@@ -1151,15 +2727,45 @@ Module Contents
column names or expressions.
:param null_treatment: Assign whether to respect or ignore null values.
- For example::
-
- df.aggregate([], last_value(col("a"), order_by="ts"))
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [10, 20, 30]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.last_value(
+ ... dfn.col("a")
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 30
+
+ >>> df = ctx.from_pydict({"a": [None, 20, 10]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.last_value(
+ ... dfn.col("a"),
+ ... filter=dfn.col("a") > dfn.lit(10),
+ ... order_by="a",
+ ... null_treatment=dfn.common.NullTreatment.IGNORE_NULLS,
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 20
.. py:function:: lcm(x: datafusion.expr.Expr, y: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the least common multiple.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [4], "b": [6]})
+ >>> result = df.select(
+ ... dfn.functions.lcm(dfn.col("a"), dfn.col("b")).alias("lcm")
+ ... )
+ >>> result.collect_column("lcm")[0].as_py()
+ 12
+
.. py:function:: lead(arg: datafusion.expr.Expr, shift_offset: int = 1, default_value: Any | None = None, partition_by: list[datafusion.expr.Expr] | datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None) -> datafusion.expr.Expr
@@ -1192,192 +2798,339 @@ Module Contents
:param order_by: Set ordering within the window frame. Accepts
column names or expressions.
- For example::
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> result = df.select(
+ ... dfn.col("a"),
+ ... dfn.functions.lead(
+ ... dfn.col("a"), shift_offset=1,
+ ... default_value=0, order_by="a"
+ ... ).alias("lead"))
+ >>> result.sort(dfn.col("a")).collect_column("lead").to_pylist()
+ [2, 3, 0]
+
+ >>> df = ctx.from_pydict({"g": ["a", "a", "b"], "v": [1, 2, 3]})
+ >>> result = df.select(
+ ... dfn.col("g"), dfn.col("v"),
+ ... dfn.functions.lead(
+ ... dfn.col("v"), shift_offset=1, default_value=0,
+ ... partition_by=dfn.col("g"), order_by="v",
+ ... ).alias("lead"))
+ >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("lead").to_pylist()
+ [2, 0, 0]
+
+
+.. py:function:: least(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the least value from a list of expressions.
- lead(col("b"), order_by="ts")
+ Returns NULL if all expressions are NULL.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 3], "b": [2, 1]})
+ >>> result = df.select(
+ ... dfn.functions.least(dfn.col("a"), dfn.col("b")).alias("least"))
+ >>> result.collect_column("least")[0].as_py()
+ 1
+ >>> result.collect_column("least")[1].as_py()
+ 1
.. py:function:: left(string: datafusion.expr.Expr, n: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the first ``n`` characters in the ``string``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["the cat"]})
+ >>> left_df = df.select(
+ ... dfn.functions.left(dfn.col("a"), dfn.lit(3)).alias("left"))
+ >>> left_df.collect_column("left")[0].as_py()
+ 'the'
+
.. py:function:: length(string: datafusion.expr.Expr) -> datafusion.expr.Expr
The number of characters in the ``string``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(dfn.functions.length(dfn.col("a")).alias("len"))
+ >>> result.collect_column("len")[0].as_py()
+ 5
+
.. py:function:: levenshtein(string1: datafusion.expr.Expr, string2: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the Levenshtein distance between the two given strings.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["kitten"]})
+ >>> result = df.select(
+ ... dfn.functions.levenshtein(dfn.col("a"), dfn.lit("sitting")).alias("d"))
+ >>> result.collect_column("d")[0].as_py()
+ 3
+
+
+.. py:function:: list_any_value(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the first non-null element in the array.
+
+ .. seealso:: This is an alias for :py:func:`array_any_value`.
+
.. py:function:: list_append(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
Appends an element to the end of an array.
- This is an alias for :py:func:`array_append`.
+ .. seealso:: This is an alias for :py:func:`array_append`.
.. py:function:: list_cat(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
Concatenates the input arrays.
- This is an alias for :py:func:`array_concat`, :py:func:`array_cat`.
+ .. seealso:: This is an alias for :py:func:`array_concat`, :py:func:`array_cat`.
.. py:function:: list_concat(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
Concatenates the input arrays.
- This is an alias for :py:func:`array_concat`, :py:func:`array_cat`.
+ .. seealso:: This is an alias for :py:func:`array_concat`, :py:func:`array_cat`.
+
+
+.. py:function:: list_contains(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns true if the element appears in the array, otherwise false.
+
+ .. seealso:: This is an alias for :py:func:`array_has`.
.. py:function:: list_dims(array: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns an array of the array's dimensions.
- This is an alias for :py:func:`array_dims`.
+ .. seealso:: This is an alias for :py:func:`array_dims`.
+
+
+.. py:function:: list_distance(array1: datafusion.expr.Expr, array2: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the Euclidean distance between two numeric arrays.
+
+ .. seealso:: This is an alias for :py:func:`array_distance`.
.. py:function:: list_distinct(array: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns distinct values from the array after removing duplicates.
- This is an alias for :py:func:`array_distinct`.
+ .. seealso:: This is an alias for :py:func:`array_distinct`.
.. py:function:: list_element(array: datafusion.expr.Expr, n: datafusion.expr.Expr) -> datafusion.expr.Expr
Extracts the element with the index n from the array.
- This is an alias for :py:func:`array_element`.
+ .. seealso:: This is an alias for :py:func:`array_element`.
+
+
+.. py:function:: list_empty(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns a boolean indicating whether the array is empty.
+
+ .. seealso:: This is an alias for :py:func:`array_empty`.
.. py:function:: list_except(array1: datafusion.expr.Expr, array2: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the elements that appear in ``array1`` but not in the ``array2``.
- This is an alias for :py:func:`array_except`.
+ .. seealso:: This is an alias for :py:func:`array_except`.
.. py:function:: list_extract(array: datafusion.expr.Expr, n: datafusion.expr.Expr) -> datafusion.expr.Expr
Extracts the element with the index n from the array.
- This is an alias for :py:func:`array_element`.
+ .. seealso:: This is an alias for :py:func:`array_element`.
+
+
+.. py:function:: list_has(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns true if the element appears in the array, otherwise false.
+
+ .. seealso:: This is an alias for :py:func:`array_has`.
+
+
+.. py:function:: list_has_all(first_array: datafusion.expr.Expr, second_array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Determines if there is complete overlap ``second_array`` in ``first_array``.
+
+ .. seealso:: This is an alias for :py:func:`array_has_all`.
+
+
+.. py:function:: list_has_any(first_array: datafusion.expr.Expr, second_array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Determine if there is an overlap between ``first_array`` and ``second_array``.
+
+ .. seealso:: This is an alias for :py:func:`array_has_any`.
.. py:function:: list_indexof(array: datafusion.expr.Expr, element: datafusion.expr.Expr, index: int | None = 1) -> datafusion.expr.Expr
Return the position of the first occurrence of ``element`` in ``array``.
- This is an alias for :py:func:`array_position`.
+ .. seealso:: This is an alias for :py:func:`array_position`.
.. py:function:: list_intersect(array1: datafusion.expr.Expr, array2: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns an the intersection of ``array1`` and ``array2``.
- This is an alias for :py:func:`array_intersect`.
+ .. seealso:: This is an alias for :py:func:`array_intersect`.
.. py:function:: list_join(expr: datafusion.expr.Expr, delimiter: datafusion.expr.Expr) -> datafusion.expr.Expr
Converts each element to its text representation.
- This is an alias for :py:func:`array_to_string`.
+ .. seealso:: This is an alias for :py:func:`array_to_string`.
.. py:function:: list_length(array: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the length of the array.
- This is an alias for :py:func:`array_length`.
+ .. seealso:: This is an alias for :py:func:`array_length`.
+
+
+.. py:function:: list_max(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the maximum value in the array.
+
+ .. seealso:: This is an alias for :py:func:`array_max`.
+
+
+.. py:function:: list_min(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the minimum value in the array.
+
+ .. seealso:: This is an alias for :py:func:`array_min`.
.. py:function:: list_ndims(array: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the number of dimensions of the array.
- This is an alias for :py:func:`array_ndims`.
+ .. seealso:: This is an alias for :py:func:`array_ndims`.
+
+
+.. py:function:: list_overlap(first_array: datafusion.expr.Expr, second_array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns true if any element appears in both arrays.
+
+ .. seealso:: This is an alias for :py:func:`array_has_any`.
+
+
+.. py:function:: list_pop_back(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the array without the last element.
+
+ .. seealso:: This is an alias for :py:func:`array_pop_back`.
+
+
+.. py:function:: list_pop_front(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the array without the first element.
+
+ .. seealso:: This is an alias for :py:func:`array_pop_front`.
.. py:function:: list_position(array: datafusion.expr.Expr, element: datafusion.expr.Expr, index: int | None = 1) -> datafusion.expr.Expr
Return the position of the first occurrence of ``element`` in ``array``.
- This is an alias for :py:func:`array_position`.
+ .. seealso:: This is an alias for :py:func:`array_position`.
.. py:function:: list_positions(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
Searches for an element in the array and returns all occurrences.
- This is an alias for :py:func:`array_positions`.
+ .. seealso:: This is an alias for :py:func:`array_positions`.
.. py:function:: list_prepend(element: datafusion.expr.Expr, array: datafusion.expr.Expr) -> datafusion.expr.Expr
Prepends an element to the beginning of an array.
- This is an alias for :py:func:`array_prepend`.
+ .. seealso:: This is an alias for :py:func:`array_prepend`.
.. py:function:: list_push_back(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
Appends an element to the end of an array.
- This is an alias for :py:func:`array_append`.
+ .. seealso:: This is an alias for :py:func:`array_append`.
.. py:function:: list_push_front(element: datafusion.expr.Expr, array: datafusion.expr.Expr) -> datafusion.expr.Expr
Prepends an element to the beginning of an array.
- This is an alias for :py:func:`array_prepend`.
+ .. seealso:: This is an alias for :py:func:`array_prepend`.
.. py:function:: list_remove(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
Removes the first element from the array equal to the given value.
- This is an alias for :py:func:`array_remove`.
+ .. seealso:: This is an alias for :py:func:`array_remove`.
.. py:function:: list_remove_all(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
Removes all elements from the array equal to the given value.
- This is an alias for :py:func:`array_remove_all`.
+ .. seealso:: This is an alias for :py:func:`array_remove_all`.
.. py:function:: list_remove_n(array: datafusion.expr.Expr, element: datafusion.expr.Expr, max: datafusion.expr.Expr) -> datafusion.expr.Expr
Removes the first ``max`` elements from the array equal to the given value.
- This is an alias for :py:func:`array_remove_n`.
+ .. seealso:: This is an alias for :py:func:`array_remove_n`.
.. py:function:: list_repeat(element: datafusion.expr.Expr, count: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns an array containing ``element`` ``count`` times.
- This is an alias for :py:func:`array_repeat`.
+ .. seealso:: This is an alias for :py:func:`array_repeat`.
.. py:function:: list_replace(array: datafusion.expr.Expr, from_val: datafusion.expr.Expr, to_val: datafusion.expr.Expr) -> datafusion.expr.Expr
Replaces the first occurrence of ``from_val`` with ``to_val``.
- This is an alias for :py:func:`array_replace`.
+ .. seealso:: This is an alias for :py:func:`array_replace`.
.. py:function:: list_replace_all(array: datafusion.expr.Expr, from_val: datafusion.expr.Expr, to_val: datafusion.expr.Expr) -> datafusion.expr.Expr
Replaces all occurrences of ``from_val`` with ``to_val``.
- This is an alias for :py:func:`array_replace_all`.
+ .. seealso:: This is an alias for :py:func:`array_replace_all`.
.. py:function:: list_replace_n(array: datafusion.expr.Expr, from_val: datafusion.expr.Expr, to_val: datafusion.expr.Expr, max: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -1387,7 +3140,7 @@ Module Contents
Replaces the first ``max`` occurrences of the specified element with another
specified element.
- This is an alias for :py:func:`array_replace_n`.
+ .. seealso:: This is an alias for :py:func:`array_replace_n`.
.. py:function:: list_resize(array: datafusion.expr.Expr, size: datafusion.expr.Expr, value: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -1395,26 +3148,37 @@ Module Contents
Returns an array with the specified size filled.
If ``size`` is greater than the ``array`` length, the additional entries will be
- filled with the given ``value``. This is an alias for :py:func:`array_resize`.
+ filled with the given ``value``.
+
+ .. seealso:: This is an alias for :py:func:`array_resize`.
+
+
+.. py:function:: list_reverse(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Reverses the order of elements in the array.
+
+ .. seealso:: This is an alias for :py:func:`array_reverse`.
.. py:function:: list_slice(array: datafusion.expr.Expr, begin: datafusion.expr.Expr, end: datafusion.expr.Expr, stride: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Returns a slice of the array.
- This is an alias for :py:func:`array_slice`.
+ .. seealso:: This is an alias for :py:func:`array_slice`.
.. py:function:: list_sort(array: datafusion.expr.Expr, descending: bool = False, null_first: bool = False) -> datafusion.expr.Expr
- This is an alias for :py:func:`array_sort`.
+ Sorts the array.
+
+ .. seealso:: This is an alias for :py:func:`array_sort`.
.. py:function:: list_to_string(expr: datafusion.expr.Expr, delimiter: datafusion.expr.Expr) -> datafusion.expr.Expr
Converts each element to its text representation.
- This is an alias for :py:func:`array_to_string`.
+ .. seealso:: This is an alias for :py:func:`array_to_string`.
.. py:function:: list_union(array1: datafusion.expr.Expr, array2: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -1423,33 +3187,82 @@ Module Contents
Duplicate rows will not be returned.
- This is an alias for :py:func:`array_union`.
+ .. seealso:: This is an alias for :py:func:`array_union`.
+
+
+.. py:function:: list_zip(*arrays: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Combines multiple arrays into a single array of structs.
+
+ .. seealso:: This is an alias for :py:func:`arrays_zip`.
.. py:function:: ln(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the natural logarithm (base e) of the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0]})
+ >>> result = df.select(dfn.functions.ln(dfn.col("a")).alias("ln"))
+ >>> result.collect_column("ln")[0].as_py()
+ 0.0
+
.. py:function:: log(base: datafusion.expr.Expr, num: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the logarithm of a number for a particular ``base``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [100.0]})
+ >>> result = df.select(
+ ... dfn.functions.log(dfn.lit(10.0), dfn.col("a")).alias("log")
+ ... )
+ >>> result.collect_column("log")[0].as_py()
+ 2.0
+
.. py:function:: log10(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Base 10 logarithm of the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [100.0]})
+ >>> result = df.select(dfn.functions.log10(dfn.col("a")).alias("log10"))
+ >>> result.collect_column("log10")[0].as_py()
+ 2.0
+
.. py:function:: log2(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Base 2 logarithm of the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [8.0]})
+ >>> result = df.select(dfn.functions.log2(dfn.col("a")).alias("log2"))
+ >>> result.collect_column("log2")[0].as_py()
+ 3.0
+
.. py:function:: lower(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Converts a string to lowercase.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["THE CaT"]})
+ >>> lower_df = df.select(dfn.functions.lower(dfn.col("a")).alias("lower"))
+ >>> lower_df.collect_column("lower")[0].as_py()
+ 'the cat'
+
.. py:function:: lpad(string: datafusion.expr.Expr, count: datafusion.expr.Expr, characters: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1459,27 +3272,205 @@ Module Contents
space by default). If the string is already longer than length then it is
truncated (on the right).
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["the cat", "a hat"]})
+ >>> lpad_df = df.select(
+ ... dfn.functions.lpad(
+ ... dfn.col("a"), dfn.lit(6)
+ ... ).alias("lpad"))
+ >>> lpad_df.collect_column("lpad")[0].as_py()
+ 'the ca'
+ >>> lpad_df.collect_column("lpad")[1].as_py()
+ ' a hat'
+
+ >>> result = df.select(
+ ... dfn.functions.lpad(
+ ... dfn.col("a"), dfn.lit(10), characters=dfn.lit(".")
+ ... ).alias("lpad"))
+ >>> result.collect_column("lpad")[0].as_py()
+ '...the cat'
+
.. py:function:: ltrim(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Removes all characters, spaces by default, from the beginning of a string.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [" a "]})
+ >>> trim_df = df.select(dfn.functions.ltrim(dfn.col("a")).alias("trimmed"))
+ >>> trim_df.collect_column("trimmed")[0].as_py()
+ 'a '
+
.. py:function:: make_array(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns an array using the specified input expressions.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> result = df.select(
+ ... dfn.functions.make_array(
+ ... dfn.lit(1), dfn.lit(2), dfn.lit(3)
+ ... ).alias("arr"))
+ >>> result.collect_column("arr")[0].as_py()
+ [1, 2, 3]
+
.. py:function:: make_date(year: datafusion.expr.Expr, month: datafusion.expr.Expr, day: datafusion.expr.Expr) -> datafusion.expr.Expr
Make a date from year, month and day component parts.
+ .. rubric:: Examples
+
+ >>> from datetime import date
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"y": [2024], "m": [1], "d": [15]})
+ >>> result = df.select(
+ ... dfn.functions.make_date(dfn.col("y"), dfn.col("m"),
+ ... dfn.col("d")).alias("dt"))
+ >>> result.collect_column("dt")[0].as_py()
+ datetime.date(2024, 1, 15)
+
.. py:function:: make_list(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns an array using the specified input expressions.
- This is an alias for :py:func:`make_array`.
+ .. seealso:: This is an alias for :py:func:`make_array`.
+
+
+.. py:function:: make_map(*args: Any) -> datafusion.expr.Expr
+
+ Returns a map expression.
+
+ Supports three calling conventions:
+
+ - ``make_map({"a": 1, "b": 2})`` — from a Python dictionary.
+ - ``make_map([keys], [values])`` — from a list of keys and a list of
+ their associated values. Both lists must be the same length.
+ - ``make_map(k1, v1, k2, v2, ...)`` — from alternating keys and their
+ associated values.
+
+ Keys and values that are not already :py:class:`~datafusion.expr.Expr`
+ are automatically converted to literal expressions.
+
+ .. rubric:: Examples
+
+ From a dictionary:
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> result = df.select(
+ ... dfn.functions.make_map({"a": 1, "b": 2}).alias("m"))
+ >>> result.collect_column("m")[0].as_py()
+ [('a', 1), ('b', 2)]
+
+ From two lists:
+
+ >>> df = ctx.from_pydict({"key": ["x", "y"], "val": [10, 20]})
+ >>> df = df.select(
+ ... dfn.functions.make_map(
+ ... [dfn.col("key")], [dfn.col("val")]
+ ... ).alias("m"))
+ >>> df.collect_column("m")[0].as_py()
+ [('x', 10)]
+
+ From alternating keys and values:
+
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> result = df.select(
+ ... dfn.functions.make_map("x", 1, "y", 2).alias("m"))
+ >>> result.collect_column("m")[0].as_py()
+ [('x', 1), ('y', 2)]
+
+
+.. py:function:: make_time(hour: datafusion.expr.Expr, minute: datafusion.expr.Expr, second: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Make a time from hour, minute and second component parts.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"h": [12], "m": [30], "s": [0]})
+ >>> result = df.select(
+ ... dfn.functions.make_time(dfn.col("h"), dfn.col("m"),
+ ... dfn.col("s")).alias("t"))
+ >>> result.collect_column("t")[0].as_py()
+ datetime.time(12, 30)
+
+
+.. py:function:: map_entries(map: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns a list of all entries (key-value struct pairs) in the map.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> df = df.select(
+ ... dfn.functions.make_map({"x": 1, "y": 2}).alias("m"))
+ >>> result = df.select(
+ ... dfn.functions.map_entries(dfn.col("m")).alias("entries"))
+ >>> result.collect_column("entries")[0].as_py()
+ [{'key': 'x', 'value': 1}, {'key': 'y', 'value': 2}]
+
+
+.. py:function:: map_extract(map: datafusion.expr.Expr, key: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the value for a given key in the map.
+
+ Returns ``[None]`` if the key is absent.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> df = df.select(
+ ... dfn.functions.make_map({"x": 1, "y": 2}).alias("m"))
+ >>> result = df.select(
+ ... dfn.functions.map_extract(
+ ... dfn.col("m"), dfn.lit("x")
+ ... ).alias("val"))
+ >>> result.collect_column("val")[0].as_py()
+ [1]
+
+
+.. py:function:: map_keys(map: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns a list of all keys in the map.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> df = df.select(
+ ... dfn.functions.make_map({"x": 1, "y": 2}).alias("m"))
+ >>> result = df.select(
+ ... dfn.functions.map_keys(dfn.col("m")).alias("keys"))
+ >>> result.collect_column("keys")[0].as_py()
+ ['x', 'y']
+
+
+.. py:function:: map_values(map: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns a list of all values in the map.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> df = df.select(
+ ... dfn.functions.make_map({"x": 1, "y": 2}).alias("m"))
+ >>> result = df.select(
+ ... dfn.functions.map_values(dfn.col("m")).alias("vals"))
+ >>> result.collect_column("vals")[0].as_py()
+ [1, 2]
.. py:function:: max(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1492,17 +3483,44 @@ Module Contents
:param expression: The value to find the maximum of
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.max(
+ ... dfn.col("a")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 3
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.max(
+ ... dfn.col("a"),
+ ... filter=dfn.col("a") < dfn.lit(3)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2
+
.. py:function:: md5(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Computes an MD5 128-bit checksum for a string expression.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(dfn.functions.md5(dfn.col("a")).alias("md5"))
+ >>> result.collect_column("md5")[0].as_py()
+ '5d41402abc4b2a76b9719d911017c592'
+
.. py:function:: mean(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Returns the average (mean) value of the argument.
- This is an alias for :py:func:`avg`.
+ .. seealso:: This is an alias for :py:func:`avg`.
.. py:function:: median(expression: datafusion.expr.Expr, distinct: bool = False, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1519,6 +3537,26 @@ Module Contents
:param distinct: If True, a single entry for each distinct value will be in the result
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.median(
+ ... dfn.col("a")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2.0
+
+ >>> df = ctx.from_pydict({"a": [1.0, 1.0, 2.0, 3.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.median(
+ ... dfn.col("a"), distinct=True,
+ ... filter=dfn.col("a") < dfn.lit(3.0),
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 1.5
+
.. py:function:: min(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1530,16 +3568,58 @@ Module Contents
:param expression: The value to find the minimum of
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.min(
+ ... dfn.col("a")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 1
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.min(
+ ... dfn.col("a"),
+ ... filter=dfn.col("a") > dfn.lit(1)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2
+
.. py:function:: named_struct(name_pairs: list[tuple[str, datafusion.expr.Expr]]) -> datafusion.expr.Expr
Returns a struct with the given names and arguments pairs.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> result = df.select(
+ ... dfn.functions.named_struct(
+ ... [("x", dfn.lit(10)), ("y", dfn.lit(20))]
+ ... ).alias("s")
+ ... )
+ >>> result.collect_column("s")[0].as_py() == {"x": 10, "y": 20}
+ True
+
.. py:function:: nanvl(x: datafusion.expr.Expr, y: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns ``x`` if ``x`` is not ``NaN``. Otherwise returns ``y``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [np.nan, 1.0], "b": [0.0, 0.0]})
+ >>> nanvl_df = df.select(
+ ... dfn.functions.nanvl(dfn.col("a"), dfn.col("b")).alias("nanvl"))
+ >>> nanvl_df.collect_column("nanvl")[0].as_py()
+ 0.0
+ >>> nanvl_df.collect_column("nanvl")[1].as_py()
+ 1.0
+
.. py:function:: now() -> datafusion.expr.Expr
@@ -1547,6 +3627,20 @@ Module Contents
This will use the same value for all instances of now() in same statement.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> result = df.select(
+ ... dfn.functions.now().alias("now")
+ ... )
+
+ Use .value instead of .as_py() because nanosecond timestamps
+ require pandas to convert to Python datetime objects.
+
+ >>> result.collect_column("now")[0].value > 0
+ True
+
.. py:function:: nth_value(expression: datafusion.expr.Expr, n: int, filter: datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None, null_treatment: datafusion.common.NullTreatment = NullTreatment.RESPECT_NULLS) -> datafusion.expr.Expr
@@ -1564,9 +3658,28 @@ Module Contents
column names or expressions.
:param null_treatment: Assign whether to respect or ignore null values.
- For example::
-
- df.aggregate([], nth_value(col("a"), 2, order_by="ts"))
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [10, 20, 30]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.nth_value(
+ ... dfn.col("a"), 1
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 10
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.nth_value(
+ ... dfn.col("a"), 1,
+ ... filter=dfn.col("a") > dfn.lit(10),
+ ... order_by="a",
+ ... null_treatment=dfn.common.NullTreatment.IGNORE_NULLS,
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 20
.. py:function:: ntile(groups: int, partition_by: list[datafusion.expr.Expr] | datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None) -> datafusion.expr.Expr
@@ -1594,9 +3707,27 @@ Module Contents
:param order_by: Set ordering within the window frame. Accepts
column names or expressions.
- For example::
-
- ntile(3, order_by="points")
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [10, 20, 30, 40]})
+ >>> result = df.select(
+ ... dfn.col("a"),
+ ... dfn.functions.ntile(
+ ... 2, order_by="a"
+ ... ).alias("nt"))
+ >>> result.sort(dfn.col("a")).collect_column("nt").to_pylist()
+ [1, 1, 2, 2]
+
+ >>> df = ctx.from_pydict(
+ ... {"g": ["a", "a", "b", "b"], "v": [1, 2, 3, 4]})
+ >>> result = df.select(
+ ... dfn.col("g"), dfn.col("v"),
+ ... dfn.functions.ntile(
+ ... 2, partition_by=dfn.col("g"), order_by="v",
+ ... ).alias("nt"))
+ >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("nt").to_pylist()
+ [1, 2, 1, 2]
.. py:function:: nullif(expr1: datafusion.expr.Expr, expr2: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -1605,21 +3736,80 @@ Module Contents
This can be used to perform the inverse operation of the COALESCE expression.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2], "b": [1, 3]})
+ >>> result = df.select(
+ ... dfn.functions.nullif(dfn.col("a"), dfn.col("b")).alias("nullif"))
+ >>> result.collect_column("nullif").to_pylist()
+ [None, 2]
+
.. py:function:: nvl(x: datafusion.expr.Expr, y: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns ``x`` if ``x`` is not ``NULL``. Otherwise returns ``y``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [None, 1], "b": [0, 0]})
+ >>> nvl_df = df.select(
+ ... dfn.functions.nvl(dfn.col("a"), dfn.col("b")).alias("nvl")
+ ... )
+ >>> nvl_df.collect_column("nvl")[0].as_py()
+ 0
+ >>> nvl_df.collect_column("nvl")[1].as_py()
+ 1
+
+
+.. py:function:: nvl2(x: datafusion.expr.Expr, y: datafusion.expr.Expr, z: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns ``y`` if ``x`` is not NULL. Otherwise returns ``z``.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [None, 1], "b": [10, 20], "c": [30, 40]})
+ >>> result = df.select(
+ ... dfn.functions.nvl2(
+ ... dfn.col("a"), dfn.col("b"), dfn.col("c")).alias("nvl2")
+ ... )
+ >>> result.collect_column("nvl2")[0].as_py()
+ 30
+ >>> result.collect_column("nvl2")[1].as_py()
+ 20
+
.. py:function:: octet_length(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the number of bytes of a string.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(dfn.functions.octet_length(dfn.col("a")).alias("len"))
+ >>> result.collect_column("len")[0].as_py()
+ 5
+
.. py:function:: order_by(expr: datafusion.expr.Expr, ascending: bool = True, nulls_first: bool = True) -> datafusion.expr.SortExpr
Creates a new sort expression.
+ .. rubric:: Examples
+
+ >>> sort_expr = dfn.functions.order_by(
+ ... dfn.col("a"), ascending=False)
+ >>> sort_expr.ascending()
+ False
+
+ >>> sort_expr = dfn.functions.order_by(
+ ... dfn.col("a"), ascending=True, nulls_first=False)
+ >>> sort_expr.nulls_first()
+ False
+
.. py:function:: overlay(string: datafusion.expr.Expr, substring: datafusion.expr.Expr, start: datafusion.expr.Expr, length: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1628,6 +3818,16 @@ Module Contents
Replace the substring of string that starts at the ``start``'th character and
extends for ``length`` characters with new substring.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["abcdef"]})
+ >>> result = df.select(
+ ... dfn.functions.overlay(dfn.col("a"), dfn.lit("XY"), dfn.lit(3),
+ ... dfn.lit(2)).alias("o"))
+ >>> result.collect_column("o")[0].as_py()
+ 'abXYef'
+
.. py:function:: percent_rank(partition_by: list[datafusion.expr.Expr] | datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None) -> datafusion.expr.Expr
@@ -1651,42 +3851,153 @@ Module Contents
:param order_by: Set ordering within the window frame. Accepts
column names or expressions.
- For example::
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [10, 20, 30]})
+ >>> result = df.select(
+ ... dfn.col("a"),
+ ... dfn.functions.percent_rank(
+ ... order_by="a"
+ ... ).alias("pr"))
+ >>> result.sort(dfn.col("a")).collect_column("pr").to_pylist()
+ [0.0, 0.5, 1.0]
+
+ >>> df = ctx.from_pydict(
+ ... {"g": ["a", "a", "a", "b", "b"], "v": [1, 2, 3, 4, 5]})
+ >>> result = df.select(
+ ... dfn.col("g"), dfn.col("v"),
+ ... dfn.functions.percent_rank(
+ ... partition_by=dfn.col("g"), order_by="v",
+ ... ).alias("pr"))
+ >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("pr").to_pylist()
+ [0.0, 0.5, 1.0, 0.0, 1.0]
- percent_rank(order_by="points")
+
+.. py:function:: percentile_cont(sort_expression: datafusion.expr.Expr | datafusion.expr.SortExpr, percentile: float, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
+
+ Computes the exact percentile of input values using continuous interpolation.
+
+ Unlike :py:func:`approx_percentile_cont`, this function computes the exact
+ percentile value rather than an approximation.
+
+ If using the builder functions described in ref:`_aggregation` this function ignores
+ the options ``order_by``, ``null_treatment``, and ``distinct``.
+
+ :param sort_expression: Values for which to find the percentile
+ :param percentile: This must be between 0.0 and 1.0, inclusive
+ :param filter: If provided, only compute against rows for which the filter is True
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0, 4.0, 5.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.percentile_cont(
+ ... dfn.col("a"), 0.5
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 3.0
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.percentile_cont(
+ ... dfn.col("a"), 0.5,
+ ... filter=dfn.col("a") > dfn.lit(1.0),
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 3.5
.. py:function:: pi() -> datafusion.expr.Expr
Returns an approximate value of π.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> from math import pi
+ >>> result = df.select(
+ ... dfn.functions.pi().alias("pi")
+ ... )
+ >>> result.collect_column("pi")[0].as_py() == pi
+ True
+
.. py:function:: pow(base: datafusion.expr.Expr, exponent: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns ``base`` raised to the power of ``exponent``.
- This is an alias of :py:func:`power`.
+ .. seealso:: This is an alias of :py:func:`power`.
.. py:function:: power(base: datafusion.expr.Expr, exponent: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns ``base`` raised to the power of ``exponent``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [2.0]})
+ >>> result = df.select(
+ ... dfn.functions.power(dfn.col("a"), dfn.lit(3.0)).alias("pow")
+ ... )
+ >>> result.collect_column("pow")[0].as_py()
+ 8.0
+
+
+.. py:function:: quantile_cont(sort_expression: datafusion.expr.Expr | datafusion.expr.SortExpr, percentile: float, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
+
+ Computes the exact percentile of input values using continuous interpolation.
+
+ .. seealso:: This is an alias for :py:func:`percentile_cont`.
+
.. py:function:: radians(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Converts the argument from degrees to radians.
+ .. rubric:: Examples
+
+ >>> from math import pi
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [180.0]})
+ >>> result = df.select(
+ ... dfn.functions.radians(dfn.col("a")).alias("rad")
+ ... )
+ >>> result.collect_column("rad")[0].as_py() == pi
+ True
+
.. py:function:: random() -> datafusion.expr.Expr
Returns a random value in the range ``0.0 <= x < 1.0``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> result = df.select(
+ ... dfn.functions.random().alias("r")
+ ... )
+ >>> val = result.collect_column("r")[0].as_py()
+ >>> 0.0 <= val < 1.0
+ True
+
.. py:function:: range(start: datafusion.expr.Expr, stop: datafusion.expr.Expr, step: datafusion.expr.Expr) -> datafusion.expr.Expr
Create a list of values in the range between start and stop.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> result = df.select(
+ ... dfn.functions.range(dfn.lit(0), dfn.lit(5), dfn.lit(2)).alias("r"))
+ >>> result.collect_column("r")[0].as_py()
+ [0, 2, 4]
+
.. py:function:: rank(partition_by: list[datafusion.expr.Expr] | datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None) -> datafusion.expr.Expr
@@ -1714,9 +4025,28 @@ Module Contents
:param order_by: Set ordering within the window frame. Accepts
column names or expressions.
- For example::
-
- rank(order_by="points")
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [10, 10, 20]})
+ >>> result = df.select(
+ ... dfn.col("a"),
+ ... dfn.functions.rank(
+ ... order_by="a"
+ ... ).alias("rnk")
+ ... )
+ >>> result.sort(dfn.col("a")).collect_column("rnk").to_pylist()
+ [1, 1, 3]
+
+ >>> df = ctx.from_pydict(
+ ... {"g": ["a", "a", "b", "b"], "v": [1, 1, 2, 3]})
+ >>> result = df.select(
+ ... dfn.col("g"), dfn.col("v"),
+ ... dfn.functions.rank(
+ ... partition_by=dfn.col("g"), order_by="v",
+ ... ).alias("rnk"))
+ >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("rnk").to_pylist()
+ [1, 1, 1, 2]
.. py:function:: regexp_count(string: datafusion.expr.Expr, pattern: datafusion.expr.Expr, start: datafusion.expr.Expr | None = None, flags: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1726,15 +4056,76 @@ Module Contents
Optional start position (the first position is 1) to search for the regular
expression.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["abcabc"]})
+ >>> result = df.select(
+ ... dfn.functions.regexp_count(
+ ... dfn.col("a"), dfn.lit("abc")
+ ... ).alias("c"))
+ >>> result.collect_column("c")[0].as_py()
+ 2
+
+ Use ``start`` to begin searching from a position, and
+ ``flags`` for case-insensitive matching:
+
+ >>> result = df.select(
+ ... dfn.functions.regexp_count(
+ ... dfn.col("a"), dfn.lit("ABC"),
+ ... start=dfn.lit(4), flags=dfn.lit("i"),
+ ... ).alias("c"))
+ >>> result.collect_column("c")[0].as_py()
+ 1
+
.. py:function:: regexp_instr(values: datafusion.expr.Expr, regex: datafusion.expr.Expr, start: datafusion.expr.Expr | None = None, n: datafusion.expr.Expr | None = None, flags: datafusion.expr.Expr | None = None, sub_expr: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Returns the position of a regular expression match in a string.
- Searches ``values`` for the ``n``-th occurrence of ``regex``, starting at position
- ``start`` (the first position is 1). Returns the starting or ending position based
- on ``end_position``. Use ``flags`` to control regex behavior and ``sub_expr`` to
- return the position of a specific capture group instead of the entire match.
+ :param values: Data to search for the regular expression match.
+ :param regex: Regular expression to search for.
+ :param start: Optional position to start the search (the first position is 1).
+ :param n: Optional occurrence of the match to find (the first occurrence is 1).
+ :param flags: Optional regular expression flags to control regex behavior.
+ :param sub_expr: Optionally capture group position instead of the entire match.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello 42 world"]})
+ >>> result = df.select(
+ ... dfn.functions.regexp_instr(
+ ... dfn.col("a"), dfn.lit("\\d+")
+ ... ).alias("pos")
+ ... )
+ >>> result.collect_column("pos")[0].as_py()
+ 7
+
+ Use ``start`` to search from a position, ``n`` for the
+ nth occurrence, and ``flags`` for case-insensitive mode:
+
+ >>> df = ctx.from_pydict({"a": ["abc ABC abc"]})
+ >>> result = df.select(
+ ... dfn.functions.regexp_instr(
+ ... dfn.col("a"), dfn.lit("abc"),
+ ... start=dfn.lit(2), n=dfn.lit(1),
+ ... flags=dfn.lit("i"),
+ ... ).alias("pos")
+ ... )
+ >>> result.collect_column("pos")[0].as_py()
+ 5
+
+ Use ``sub_expr`` to get the position of a capture group:
+
+ >>> result = df.select(
+ ... dfn.functions.regexp_instr(
+ ... dfn.col("a"), dfn.lit("(abc)"),
+ ... sub_expr=dfn.lit(1),
+ ... ).alias("pos")
+ ... )
+ >>> result.collect_column("pos")[0].as_py()
+ 1
.. py:function:: regexp_like(string: datafusion.expr.Expr, regex: datafusion.expr.Expr, flags: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1744,6 +4135,29 @@ Module Contents
Tests a string using a regular expression returning true if at least one match,
false otherwise.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello123"]})
+ >>> result = df.select(
+ ... dfn.functions.regexp_like(
+ ... dfn.col("a"), dfn.lit("\\d+")
+ ... ).alias("m")
+ ... )
+ >>> result.collect_column("m")[0].as_py()
+ True
+
+ Use ``flags`` for case-insensitive matching:
+
+ >>> result = df.select(
+ ... dfn.functions.regexp_like(
+ ... dfn.col("a"), dfn.lit("HELLO"),
+ ... flags=dfn.lit("i"),
+ ... ).alias("m")
+ ... )
+ >>> result.collect_column("m")[0].as_py()
+ True
+
.. py:function:: regexp_match(string: datafusion.expr.Expr, regex: datafusion.expr.Expr, flags: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1752,6 +4166,29 @@ Module Contents
Returns an array with each element containing the leftmost-first match of the
corresponding index in ``regex`` to string in ``string``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello 42 world"]})
+ >>> result = df.select(
+ ... dfn.functions.regexp_match(
+ ... dfn.col("a"), dfn.lit("(\\d+)")
+ ... ).alias("m")
+ ... )
+ >>> result.collect_column("m")[0].as_py()
+ ['42']
+
+ Use ``flags`` for case-insensitive matching:
+
+ >>> result = df.select(
+ ... dfn.functions.regexp_match(
+ ... dfn.col("a"), dfn.lit("(HELLO)"),
+ ... flags=dfn.lit("i"),
+ ... ).alias("m")
+ ... )
+ >>> result.collect_column("m")[0].as_py()
+ ['hello']
+
.. py:function:: regexp_replace(string: datafusion.expr.Expr, pattern: datafusion.expr.Expr, replacement: datafusion.expr.Expr, flags: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1763,6 +4200,31 @@ Module Contents
Supported flags with the addition of 'g' can be found at
<https://docs.rs/regex/latest/regex/#grouping-and-flags>
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello 42"]})
+ >>> result = df.select(
+ ... dfn.functions.regexp_replace(
+ ... dfn.col("a"), dfn.lit("\\d+"),
+ ... dfn.lit("XX")
+ ... ).alias("r")
+ ... )
+ >>> result.collect_column("r")[0].as_py()
+ 'hello XX'
+
+ Use the ``g`` flag to replace all occurrences:
+
+ >>> df = ctx.from_pydict({"a": ["a1 b2 c3"]})
+ >>> result = df.select(
+ ... dfn.functions.regexp_replace(
+ ... dfn.col("a"), dfn.lit("\\d+"),
+ ... dfn.lit("X"), flags=dfn.lit("g"),
+ ... ).alias("r")
+ ... )
+ >>> result.collect_column("r")[0].as_py()
+ 'aX bX cX'
+
.. py:function:: regr_avgx(y: datafusion.expr.Expr, x: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1778,6 +4240,25 @@ Module Contents
:param x: The linear regression independent variable
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"y": [1.0, 2.0, 3.0], "x": [4.0, 5.0, 6.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_avgx(
+ ... dfn.col("y"), dfn.col("x")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 5.0
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_avgx(
+ ... dfn.col("y"), dfn.col("x"),
+ ... filter=dfn.col("y") > dfn.lit(1.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 5.5
+
.. py:function:: regr_avgy(y: datafusion.expr.Expr, x: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1793,6 +4274,25 @@ Module Contents
:param x: The linear regression independent variable
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"y": [1.0, 2.0, 3.0], "x": [4.0, 5.0, 6.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_avgy(
+ ... dfn.col("y"), dfn.col("x")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2.0
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_avgy(
+ ... dfn.col("y"), dfn.col("x"),
+ ... filter=dfn.col("y") > dfn.lit(1.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2.5
+
.. py:function:: regr_count(y: datafusion.expr.Expr, x: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1808,6 +4308,25 @@ Module Contents
:param x: The linear regression independent variable
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"y": [1.0, 2.0, 3.0], "x": [4.0, 5.0, 6.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_count(
+ ... dfn.col("y"), dfn.col("x")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 3
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_count(
+ ... dfn.col("y"), dfn.col("x"),
+ ... filter=dfn.col("y") > dfn.lit(1.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2
+
.. py:function:: regr_intercept(y: datafusion.expr.Expr, x: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1823,6 +4342,27 @@ Module Contents
:param x: The linear regression independent variable
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"y": [2.0, 4.0, 6.0], "x": [4.0, 16.0, 36.0]})
+ >>> result = df.aggregate(
+ ... [],
+ ... [dfn.functions.regr_intercept(
+ ... dfn.col("y"), dfn.col("x")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 1.714...
+
+ >>> result = df.aggregate(
+ ... [],
+ ... [dfn.functions.regr_intercept(
+ ... dfn.col("y"), dfn.col("x"),
+ ... filter=dfn.col("y") > dfn.lit(2.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2.4
+
.. py:function:: regr_r2(y: datafusion.expr.Expr, x: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1838,6 +4378,25 @@ Module Contents
:param x: The linear regression independent variable
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"y": [2.0, 4.0, 6.0], "x": [4.0, 16.0, 36.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_r2(
+ ... dfn.col("y"), dfn.col("x")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 0.9795...
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_r2(
+ ... dfn.col("y"), dfn.col("x"),
+ ... filter=dfn.col("y") > dfn.lit(2.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 1.0
+
.. py:function:: regr_slope(y: datafusion.expr.Expr, x: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1853,6 +4412,25 @@ Module Contents
:param x: The linear regression independent variable
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"y": [2.0, 4.0, 6.0], "x": [4.0, 16.0, 36.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_slope(
+ ... dfn.col("y"), dfn.col("x")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 0.122...
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_slope(
+ ... dfn.col("y"), dfn.col("x"),
+ ... filter=dfn.col("y") > dfn.lit(2.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 0.1
+
.. py:function:: regr_sxx(y: datafusion.expr.Expr, x: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1868,6 +4446,25 @@ Module Contents
:param x: The linear regression independent variable
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"y": [1.0, 2.0, 3.0], "x": [1.0, 2.0, 3.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_sxx(
+ ... dfn.col("y"), dfn.col("x")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2.0
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_sxx(
+ ... dfn.col("y"), dfn.col("x"),
+ ... filter=dfn.col("y") > dfn.lit(1.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 0.5
+
.. py:function:: regr_sxy(y: datafusion.expr.Expr, x: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1883,6 +4480,25 @@ Module Contents
:param x: The linear regression independent variable
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"y": [1.0, 2.0, 3.0], "x": [1.0, 2.0, 3.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_sxy(
+ ... dfn.col("y"), dfn.col("x")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2.0
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_sxy(
+ ... dfn.col("y"), dfn.col("x"),
+ ... filter=dfn.col("y") > dfn.lit(1.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 0.5
+
.. py:function:: regr_syy(y: datafusion.expr.Expr, x: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1898,26 +4514,80 @@ Module Contents
:param x: The linear regression independent variable
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"y": [1.0, 2.0, 3.0], "x": [1.0, 2.0, 3.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_syy(
+ ... dfn.col("y"), dfn.col("x")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2.0
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.regr_syy(
+ ... dfn.col("y"), dfn.col("x"),
+ ... filter=dfn.col("y") > dfn.lit(1.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 0.5
+
.. py:function:: repeat(string: datafusion.expr.Expr, n: datafusion.expr.Expr) -> datafusion.expr.Expr
Repeats the ``string`` to ``n`` times.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["ha"]})
+ >>> result = df.select(
+ ... dfn.functions.repeat(dfn.col("a"), dfn.lit(3)).alias("r"))
+ >>> result.collect_column("r")[0].as_py()
+ 'hahaha'
+
.. py:function:: replace(string: datafusion.expr.Expr, from_val: datafusion.expr.Expr, to_val: datafusion.expr.Expr) -> datafusion.expr.Expr
Replaces all occurrences of ``from_val`` with ``to_val`` in the ``string``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello world"]})
+ >>> result = df.select(
+ ... dfn.functions.replace(dfn.col("a"), dfn.lit("world"),
+ ... dfn.lit("there")).alias("r"))
+ >>> result.collect_column("r")[0].as_py()
+ 'hello there'
+
.. py:function:: reverse(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Reverse the string argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(dfn.functions.reverse(dfn.col("a")).alias("r"))
+ >>> result.collect_column("r")[0].as_py()
+ 'olleh'
+
.. py:function:: right(string: datafusion.expr.Expr, n: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the last ``n`` characters in the ``string``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(dfn.functions.right(dfn.col("a"), dfn.lit(3)).alias("r"))
+ >>> result.collect_column("r")[0].as_py()
+ 'llo'
+
.. py:function:: round(value: datafusion.expr.Expr, decimal_places: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1927,6 +4597,21 @@ Module Contents
decimal places. You can specify a negative number of decimal places. For example
``round(lit(125.2345), lit(-2))`` would yield a value of ``100.0``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.567]})
+ >>> result = df.select(dfn.functions.round(dfn.col("a"), dfn.lit(2)).alias("r"))
+ >>> result.collect_column("r")[0].as_py()
+ 1.57
+
+
+.. py:function:: row(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns a struct with the given arguments.
+
+ .. seealso:: This is an alias for :py:func:`struct`.
+
.. py:function:: row_number(partition_by: list[datafusion.expr.Expr] | datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None) -> datafusion.expr.Expr
@@ -1949,9 +4634,27 @@ Module Contents
:param order_by: Set ordering within the window frame. Accepts
column names or expressions.
- For example::
-
- row_number(order_by="points")
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [10, 20, 30]})
+ >>> result = df.select(
+ ... dfn.col("a"),
+ ... dfn.functions.row_number(
+ ... order_by="a"
+ ... ).alias("rn"))
+ >>> result.sort(dfn.col("a")).collect_column("rn").to_pylist()
+ [1, 2, 3]
+
+ >>> df = ctx.from_pydict(
+ ... {"g": ["a", "a", "b", "b"], "v": [1, 2, 3, 4]})
+ >>> result = df.select(
+ ... dfn.col("g"), dfn.col("v"),
+ ... dfn.functions.row_number(
+ ... partition_by=dfn.col("g"), order_by="v",
+ ... ).alias("rn"))
+ >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("rn").to_pylist()
+ [1, 2, 1, 2]
.. py:function:: rpad(string: datafusion.expr.Expr, count: datafusion.expr.Expr, characters: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -1961,46 +4664,127 @@ Module Contents
Extends the string to length length by appending the characters fill (a space
by default). If the string is already longer than length then it is truncated.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hi"]})
+ >>> result = df.select(
+ ... dfn.functions.rpad(dfn.col("a"), dfn.lit(5), dfn.lit("!")).alias("r"))
+ >>> result.collect_column("r")[0].as_py()
+ 'hi!!!'
+
.. py:function:: rtrim(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Removes all characters, spaces by default, from the end of a string.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [" a "]})
+ >>> trim_df = df.select(dfn.functions.rtrim(dfn.col("a")).alias("trimmed"))
+ >>> trim_df.collect_column("trimmed")[0].as_py()
+ ' a'
+
.. py:function:: sha224(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Computes the SHA-224 hash of a binary string.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(
+ ... dfn.functions.sha224(dfn.col("a")).alias("h")
+ ... )
+ >>> result.collect_column("h")[0].as_py().hex()
+ 'ea09ae9cc6768c50fcee903ed054556e5bfc8347907f12598aa24193'
+
.. py:function:: sha256(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Computes the SHA-256 hash of a binary string.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(
+ ... dfn.functions.sha256(dfn.col("a")).alias("h")
+ ... )
+ >>> result.collect_column("h")[0].as_py().hex()
+ '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824'
+
.. py:function:: sha384(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Computes the SHA-384 hash of a binary string.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(
+ ... dfn.functions.sha384(dfn.col("a")).alias("h")
+ ... )
+ >>> result.collect_column("h")[0].as_py().hex()
+ '59e1748777448c69de6b800d7a33bbfb9ff1b...
+
.. py:function:: sha512(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Computes the SHA-512 hash of a binary string.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(
+ ... dfn.functions.sha512(dfn.col("a")).alias("h")
+ ... )
+ >>> result.collect_column("h")[0].as_py().hex()
+ '9b71d224bd62f3785d96d46ad3ea3d73319bfb...
+
.. py:function:: signum(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the sign of the argument (-1, 0, +1).
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [-5.0, 0.0, 5.0]})
+ >>> result = df.select(dfn.functions.signum(dfn.col("a")).alias("s"))
+ >>> result.collect_column("s").to_pylist()
+ [-1.0, 0.0, 1.0]
+
.. py:function:: sin(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the sine of the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0.0]})
+ >>> result = df.select(dfn.functions.sin(dfn.col("a")).alias("sin"))
+ >>> result.collect_column("sin")[0].as_py()
+ 0.0
+
.. py:function:: sinh(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the hyperbolic sine of the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0.0]})
+ >>> result = df.select(dfn.functions.sinh(dfn.col("a")).alias("sinh"))
+ >>> result.collect_column("sinh")[0].as_py()
+ 0.0
+
.. py:function:: split_part(string: datafusion.expr.Expr, delimiter: datafusion.expr.Expr, index: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -2009,16 +4793,44 @@ Module Contents
Splits a string based on a delimiter and picks out the desired field based
on the index.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["a,b,c"]})
+ >>> result = df.select(
+ ... dfn.functions.split_part(
+ ... dfn.col("a"), dfn.lit(","), dfn.lit(2)
+ ... ).alias("s"))
+ >>> result.collect_column("s")[0].as_py()
+ 'b'
+
.. py:function:: sqrt(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the square root of the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [9.0]})
+ >>> result = df.select(dfn.functions.sqrt(dfn.col("a")).alias("sqrt"))
+ >>> result.collect_column("sqrt")[0].as_py()
+ 3.0
+
.. py:function:: starts_with(string: datafusion.expr.Expr, prefix: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns true if string starts with prefix.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello_from_datafusion"]})
+ >>> result = df.select(
+ ... dfn.functions.starts_with(dfn.col("a"), dfn.lit("hello")).alias("sw"))
+ >>> result.collect_column("sw")[0].as_py()
+ True
+
.. py:function:: stddev(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -2030,6 +4842,25 @@ Module Contents
:param expression: The value to find the minimum of
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [2.0, 4.0, 6.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.stddev(
+ ... dfn.col("a")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2.0
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.stddev(
+ ... dfn.col("a"),
+ ... filter=dfn.col("a") > dfn.lit(2.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 1.41...
+
.. py:function:: stddev_pop(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -2041,12 +4872,34 @@ Module Contents
:param expression: The value to find the minimum of
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0.0, 1.0, 3.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.stddev_pop(
+ ... dfn.col("a")
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 1.247...
+
+ >>> df = ctx.from_pydict({"a": [0.0, 1.0, 3.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.stddev_pop(
+ ... dfn.col("a"),
+ ... filter=dfn.col("a") > dfn.lit(0.0)
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 1.0
+
.. py:function:: stddev_samp(arg: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Computes the sample standard deviation of the argument.
- This is an alias for :py:func:`stddev`.
+ .. seealso:: This is an alias for :py:func:`stddev`.
.. py:function:: string_agg(expression: datafusion.expr.Expr, delimiter: str, filter: datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None) -> datafusion.expr.Expr
@@ -2066,25 +4919,109 @@ Module Contents
:param order_by: Set the ordering of the expression to evaluate. Accepts
column names or expressions.
- For example::
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["x", "y", "z"]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.string_agg(
+ ... dfn.col("a"), ",", order_by="a"
+ ... ).alias("s")])
+ >>> result.collect_column("s")[0].as_py()
+ 'x,y,z'
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.string_agg(
+ ... dfn.col("a"), ",",
+ ... filter=dfn.col("a") > dfn.lit("x"),
+ ... order_by="a",
+ ... ).alias("s")])
+ >>> result.collect_column("s")[0].as_py()
+ 'y,z'
+
- df.aggregate([], string_agg(col("a"), ",", order_by="b"))
+.. py:function:: string_to_array(string: datafusion.expr.Expr, delimiter: datafusion.expr.Expr, null_string: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
+
+ Splits a string based on a delimiter and returns an array of parts.
+
+ Any parts matching the optional ``null_string`` will be replaced with ``NULL``.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello,world"]})
+ >>> result = df.select(
+ ... dfn.functions.string_to_array(
+ ... dfn.col("a"), dfn.lit(","),
+ ... ).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ ['hello', 'world']
+
+ Replace parts matching a ``null_string`` with ``NULL``:
+
+ >>> result = df.select(
+ ... dfn.functions.string_to_array(
+ ... dfn.col("a"), dfn.lit(","), null_string=dfn.lit("world"),
+ ... ).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ ['hello', None]
+
+
+.. py:function:: string_to_list(string: datafusion.expr.Expr, delimiter: datafusion.expr.Expr, null_string: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
+
+ Splits a string based on a delimiter and returns an array of parts.
+
+ .. seealso:: This is an alias for :py:func:`string_to_array`.
.. py:function:: strpos(string: datafusion.expr.Expr, substring: datafusion.expr.Expr) -> datafusion.expr.Expr
Finds the position from where the ``substring`` matches the ``string``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(
+ ... dfn.functions.strpos(dfn.col("a"), dfn.lit("llo")).alias("pos"))
+ >>> result.collect_column("pos")[0].as_py()
+ 3
+
.. py:function:: struct(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns a struct with the given arguments.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1], "b": [2]})
+ >>> result = df.select(
+ ... dfn.functions.struct(
+ ... dfn.col("a"), dfn.col("b")
+ ... ).alias("s")
+ ... )
+
+ Children in the new struct will always be `c0`, ..., `cN-1`
+ for `N` children.
+
+ >>> result.collect_column("s")[0].as_py() == {"c0": 1, "c1": 2}
+ True
+
.. py:function:: substr(string: datafusion.expr.Expr, position: datafusion.expr.Expr) -> datafusion.expr.Expr
Substring from the ``position`` to the end.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(
+ ... dfn.functions.substr(dfn.col("a"), dfn.lit(3)).alias("s"))
+ >>> result.collect_column("s")[0].as_py()
+ 'llo'
+
.. py:function:: substr_index(string: datafusion.expr.Expr, delimiter: datafusion.expr.Expr, count: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -2093,11 +5030,32 @@ Module Contents
The return will be the ``string`` from before ``count`` occurrences of
``delimiter``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["a.b.c"]})
+ >>> result = df.select(
+ ... dfn.functions.substr_index(dfn.col("a"), dfn.lit("."),
+ ... dfn.lit(2)).alias("s"))
+ >>> result.collect_column("s")[0].as_py()
+ 'a.b'
+
.. py:function:: substring(string: datafusion.expr.Expr, position: datafusion.expr.Expr, length: datafusion.expr.Expr) -> datafusion.expr.Expr
Substring from the ``position`` with ``length`` characters.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello world"]})
+ >>> result = df.select(
+ ... dfn.functions.substring(
+ ... dfn.col("a"), dfn.lit(1), dfn.lit(5)
+ ... ).alias("s"))
+ >>> result.collect_column("s")[0].as_py()
+ 'hello'
+
.. py:function:: sum(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -2111,21 +5069,137 @@ Module Contents
:param expression: Values to combine into an array
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.sum(
+ ... dfn.col("a")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 6
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.sum(
+ ... dfn.col("a"),
+ ... filter=dfn.col("a") > dfn.lit(1)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 5
+
.. py:function:: tan(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the tangent of the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0.0]})
+ >>> result = df.select(dfn.functions.tan(dfn.col("a")).alias("tan"))
+ >>> result.collect_column("tan")[0].as_py()
+ 0.0
+
.. py:function:: tanh(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns the hyperbolic tangent of the argument.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [0.0]})
+ >>> result = df.select(dfn.functions.tanh(dfn.col("a")).alias("tanh"))
+ >>> result.collect_column("tanh")[0].as_py()
+ 0.0
+
+
+.. py:function:: to_char(arg: datafusion.expr.Expr, formatter: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns a string representation of a date, time, timestamp or duration.
+
+ For usage of ``formatter`` see the rust chrono package ``strftime`` package.
+
+ [Documentation here.](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["2021-01-01T00:00:00"]})
+ >>> result = df.select(
+ ... dfn.functions.to_char(
+ ... dfn.functions.to_timestamp(dfn.col("a")),
+ ... dfn.lit("%Y/%m/%d"),
+ ... ).alias("formatted")
+ ... )
+ >>> result.collect_column("formatted")[0].as_py()
+ '2021/01/01'
+
+
+.. py:function:: to_date(arg: datafusion.expr.Expr, *formatters: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Converts a value to a date (YYYY-MM-DD).
+
+ Supports strings, numeric and timestamp types as input.
+ Integers and doubles are interpreted as days since the unix epoch.
+ Strings are parsed as YYYY-MM-DD (e.g. '2023-07-20')
+ if ``formatters`` are not provided.
+
+ For usage of ``formatters`` see the rust chrono package ``strftime`` package.
+
+ [Documentation here.](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["2021-07-20"]})
+ >>> result = df.select(
+ ... dfn.functions.to_date(dfn.col("a")).alias("dt"))
+ >>> str(result.collect_column("dt")[0].as_py())
+ '2021-07-20'
+
.. py:function:: to_hex(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Converts an integer to a hexadecimal string.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [255]})
+ >>> result = df.select(dfn.functions.to_hex(dfn.col("a")).alias("hex"))
+ >>> result.collect_column("hex")[0].as_py()
+ 'ff'
+
+
+.. py:function:: to_local_time(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Converts a timestamp with a timezone to a timestamp without a timezone.
+
+ This function handles daylight saving time changes.
+
+
+.. py:function:: to_time(arg: datafusion.expr.Expr, *formatters: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Converts a value to a time. Supports strings and timestamps as input.
+
+ If ``formatters`` is not provided strings are parsed as HH:MM:SS, HH:MM or
+ HH:MM:SS.nnnnnnnnn;
+
+ For usage of ``formatters`` see the rust chrono package ``strftime`` package.
+
+ [Documentation here.](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["14:30:00"]})
+ >>> result = df.select(
+ ... dfn.functions.to_time(dfn.col("a")).alias("t"))
+ >>> str(result.collect_column("t")[0].as_py())
+ '14:30:00'
+
.. py:function:: to_timestamp(arg: datafusion.expr.Expr, *formatters: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -2135,6 +5209,18 @@ Module Contents
[Documentation here.](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["2021-01-01T00:00:00"]})
+ >>> result = df.select(
+ ... dfn.functions.to_timestamp(
+ ... dfn.col("a")
+ ... ).alias("ts")
+ ... )
+ >>> str(result.collect_column("ts")[0].as_py())
+ '2021-01-01 00:00:00'
+
.. py:function:: to_timestamp_micros(arg: datafusion.expr.Expr, *formatters: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -2142,6 +5228,18 @@ Module Contents
See :py:func:`to_timestamp` for a description on how to use formatters.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["2021-01-01T00:00:00"]})
+ >>> result = df.select(
+ ... dfn.functions.to_timestamp_micros(
+ ... dfn.col("a")
+ ... ).alias("ts")
+ ... )
+ >>> str(result.collect_column("ts")[0].as_py())
+ '2021-01-01 00:00:00'
+
.. py:function:: to_timestamp_millis(arg: datafusion.expr.Expr, *formatters: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -2149,6 +5247,18 @@ Module Contents
See :py:func:`to_timestamp` for a description on how to use formatters.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["2021-01-01T00:00:00"]})
+ >>> result = df.select(
+ ... dfn.functions.to_timestamp_millis(
+ ... dfn.col("a")
+ ... ).alias("ts")
+ ... )
+ >>> str(result.collect_column("ts")[0].as_py())
+ '2021-01-01 00:00:00'
+
.. py:function:: to_timestamp_nanos(arg: datafusion.expr.Expr, *formatters: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -2156,6 +5266,18 @@ Module Contents
See :py:func:`to_timestamp` for a description on how to use formatters.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["2021-01-01T00:00:00"]})
+ >>> result = df.select(
+ ... dfn.functions.to_timestamp_nanos(
+ ... dfn.col("a")
+ ... ).alias("ts")
+ ... )
+ >>> str(result.collect_column("ts")[0].as_py())
+ '2021-01-01 00:00:00'
+
.. py:function:: to_timestamp_seconds(arg: datafusion.expr.Expr, *formatters: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -2163,42 +5285,163 @@ Module Contents
See :py:func:`to_timestamp` for a description on how to use formatters.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["2021-01-01T00:00:00"]})
+ >>> result = df.select(
+ ... dfn.functions.to_timestamp_seconds(
+ ... dfn.col("a")
+ ... ).alias("ts")
+ ... )
+ >>> str(result.collect_column("ts")[0].as_py())
+ '2021-01-01 00:00:00'
+
.. py:function:: to_unixtime(string: datafusion.expr.Expr, *format_arguments: datafusion.expr.Expr) -> datafusion.expr.Expr
Converts a string and optional formats to a Unixtime.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["1970-01-01T00:00:00"]})
+ >>> result = df.select(dfn.functions.to_unixtime(dfn.col("a")).alias("u"))
+ >>> result.collect_column("u")[0].as_py()
+ 0
+
.. py:function:: translate(string: datafusion.expr.Expr, from_val: datafusion.expr.Expr, to_val: datafusion.expr.Expr) -> datafusion.expr.Expr
Replaces the characters in ``from_val`` with the counterpart in ``to_val``.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(
+ ... dfn.functions.translate(dfn.col("a"), dfn.lit("helo"),
+ ... dfn.lit("HELO")).alias("t"))
+ >>> result.collect_column("t")[0].as_py()
+ 'HELLO'
+
.. py:function:: trim(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Removes all characters, spaces by default, from both sides of a string.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [" hello "]})
+ >>> result = df.select(dfn.functions.trim(dfn.col("a")).alias("t"))
+ >>> result.collect_column("t")[0].as_py()
+ 'hello'
+
.. py:function:: trunc(num: datafusion.expr.Expr, precision: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Truncate the number toward zero with optional precision.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.567]})
+ >>> result = df.select(
+ ... dfn.functions.trunc(
+ ... dfn.col("a")
+ ... ).alias("t"))
+ >>> result.collect_column("t")[0].as_py()
+ 1.0
+
+ >>> result = df.select(
+ ... dfn.functions.trunc(
+ ... dfn.col("a"), precision=dfn.lit(2)
+ ... ).alias("t"))
+ >>> result.collect_column("t")[0].as_py()
+ 1.56
+
+
+.. py:function:: union_extract(union_expr: datafusion.expr.Expr, field_name: datafusion.expr.Expr | str) -> datafusion.expr.Expr
+
+ Extracts a value from a union type by field name.
+
+ Returns the value of the named field if it is the currently selected
+ variant, otherwise returns NULL.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> types = pa.array([0, 1, 0], type=pa.int8())
+ >>> offsets = pa.array([0, 0, 1], type=pa.int32())
+ >>> arr = pa.UnionArray.from_dense(
+ ... types, offsets, [pa.array([1, 2]), pa.array(["hi"])],
+ ... ["int", "str"], [0, 1],
+ ... )
+ >>> batch = pa.RecordBatch.from_arrays([arr], names=["u"])
+ >>> df = ctx.create_dataframe([[batch]])
+ >>> result = df.select(
+ ... dfn.functions.union_extract(dfn.col("u"), "int").alias("val")
+ ... )
+ >>> result.collect_column("val").to_pylist()
+ [1, None, 2]
+
+
+.. py:function:: union_tag(union_expr: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the tag (active field name) of a union type.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> types = pa.array([0, 1, 0], type=pa.int8())
+ >>> offsets = pa.array([0, 0, 1], type=pa.int32())
+ >>> arr = pa.UnionArray.from_dense(
+ ... types, offsets, [pa.array([1, 2]), pa.array(["hi"])],
+ ... ["int", "str"], [0, 1],
+ ... )
+ >>> batch = pa.RecordBatch.from_arrays([arr], names=["u"])
+ >>> df = ctx.create_dataframe([[batch]])
+ >>> result = df.select(
+ ... dfn.functions.union_tag(dfn.col("u")).alias("tag")
+ ... )
+ >>> result.collect_column("tag").to_pylist()
+ ['int', 'str', 'int']
+
.. py:function:: upper(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
Converts a string to uppercase.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["hello"]})
+ >>> result = df.select(dfn.functions.upper(dfn.col("a")).alias("u"))
+ >>> result.collect_column("u")[0].as_py()
+ 'HELLO'
+
.. py:function:: uuid() -> datafusion.expr.Expr
Returns uuid v4 as a string value.
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> result = df.select(
+ ... dfn.functions.uuid().alias("u")
+ ... )
+ >>> len(result.collect_column("u")[0].as_py()) == 36
+ True
+
.. py:function:: var(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Computes the sample variance of the argument.
- This is an alias for :py:func:`var_samp`.
+ .. seealso:: This is an alias for :py:func:`var_samp`.
.. py:function:: var_pop(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -2211,6 +5454,32 @@ Module Contents
:param expression: The variable to compute the variance for
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [-1.0, 0.0, 2.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.var_pop(
+ ... dfn.col("a")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 1.555...
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.var_pop(
+ ... dfn.col("a"),
+ ... filter=dfn.col("a") > dfn.lit(-1.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 1.0
+
+
+.. py:function:: var_population(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
+
+ Computes the population variance of the argument.
+
+ .. seealso:: This is an alias for :py:func:`var_pop`.
+
.. py:function:: var_samp(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
@@ -2222,12 +5491,44 @@ Module Contents
:param expression: The variable to compute the variance for
:param filter: If provided, only compute against rows for which the filter is True
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.var_samp(
+ ... dfn.col("a")
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 1.0
+
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.var_samp(
+ ... dfn.col("a"),
+ ... filter=dfn.col("a") > dfn.lit(1.0)
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 0.5
+
.. py:function:: var_sample(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Computes the sample variance of the argument.
- This is an alias for :py:func:`var_samp`.
+ .. seealso:: This is an alias for :py:func:`var_samp`.
+
+
+.. py:function:: version() -> datafusion.expr.Expr
+
+ Returns the DataFusion version string.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.empty_table()
+ >>> result = df.select(dfn.functions.version().alias("v"))
+ >>> "Apache DataFusion" in result.collect_column("v")[0].as_py()
+ True
.. py:function:: when(when: datafusion.expr.Expr, then: datafusion.expr.Expr) -> datafusion.expr.CaseBuilder
@@ -2238,19 +5539,16 @@ Module Contents
expression ``expr``. See :py:class:`~datafusion.expr.CaseBuilder` for
detailed usage.
+ .. rubric:: Examples
-.. py:function:: window(name: str, args: list[datafusion.expr.Expr], partition_by: list[datafusion.expr.Expr] | datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None, window_frame: datafusion.expr.WindowFrame | None = None, filter: datafusion.expr.Expr | None = None, distinct: bool = False, ctx: datafusion.context.SessionContext | None = None) -> datafusion.expr.Expr
-
- Creates a new Window function expression.
-
- This interface will soon be deprecated. Instead of using this interface,
- users should call the window functions directly. For example, to perform a
- lag use::
-
- df.select(functions.lag(col("a")).partition_by(col("b")).build())
-
- The ``order_by`` parameter accepts column names or expressions, e.g.::
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> result = df.select(
+ ... dfn.functions.when(dfn.col("a") > dfn.lit(2),
+ ... dfn.lit("big")).otherwise(dfn.lit("small")).alias("c"))
+ >>> result.collect_column("c")[2].as_py()
+ 'big'
- window("lag", [col("a")], order_by="ts")
+.. py:data:: today
diff --git a/_sources/autoapi/datafusion/html_formatter/index.rst.txt b/_sources/autoapi/datafusion/html_formatter/index.rst.txt
deleted file mode 100644
index 2b9efd82..00000000
--- a/_sources/autoapi/datafusion/html_formatter/index.rst.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-datafusion.html_formatter
-=========================
-
-.. py:module:: datafusion.html_formatter
-
-.. autoapi-nested-parse::
-
- Deprecated module for dataframe formatting.
-
-
-
diff --git a/_sources/autoapi/datafusion/index.rst.txt b/_sources/autoapi/datafusion/index.rst.txt
index 24fecff4..2cccaacc 100644
--- a/_sources/autoapi/datafusion/index.rst.txt
+++ b/_sources/autoapi/datafusion/index.rst.txt
@@ -24,7 +24,6 @@ Submodules
/autoapi/datafusion/dataframe_formatter/index
/autoapi/datafusion/expr/index
/autoapi/datafusion/functions/index
- /autoapi/datafusion/html_formatter/index
/autoapi/datafusion/input/index
/autoapi/datafusion/io/index
/autoapi/datafusion/object_store/index
@@ -44,10 +43,6 @@ Attributes
datafusion.DFSchema
datafusion.col
datafusion.column
- datafusion.udaf
- datafusion.udf
- datafusion.udtf
- datafusion.udwf
Classes
@@ -55,13 +50,11 @@ Classes
.. autoapisummary::
- datafusion.Accumulator
- datafusion.AggregateUDF
datafusion.Catalog
datafusion.CsvReadOptions
datafusion.DataFrameWriteOptions
- datafusion.Database
datafusion.ExecutionPlan
+ datafusion.ExplainFormat
datafusion.Expr
datafusion.InsertOp
datafusion.LogicalPlan
@@ -71,12 +64,9 @@ Classes
datafusion.RecordBatchStream
datafusion.RuntimeEnvBuilder
datafusion.SQLOptions
- datafusion.ScalarUDF
datafusion.SessionConfig
datafusion.Table
- datafusion.TableFunction
datafusion.WindowFrame
- datafusion.WindowUDF
Functions
@@ -96,179 +86,6 @@ Functions
Package Contents
----------------
-.. py:class:: Accumulator
-
- Defines how an :py:class:`AggregateUDF` accumulates values.
-
-
- .. py:method:: evaluate() -> pyarrow.Scalar
- :abstractmethod:
-
-
- Return the resultant value.
-
- While this function template expects a PyArrow Scalar value return type,
- you can return any value that can be converted into a Scalar. This
- includes basic Python data types such as integers and strings. In
- addition to primitive types, we currently support PyArrow, nanoarrow,
- and arro3 objects in addition to primitive data types. Other objects
- that support the Arrow FFI standard will be given a "best attempt" at
- conversion to scalar objects.
-
-
-
- .. py:method:: merge(states: list[pyarrow.Array]) -> None
- :abstractmethod:
-
-
- Merge a set of states.
-
-
-
- .. py:method:: state() -> list[pyarrow.Scalar]
- :abstractmethod:
-
-
- Return the current state.
-
- While this function template expects PyArrow Scalar values return type,
- you can return any value that can be converted into a Scalar. This
- includes basic Python data types such as integers and strings. In
- addition to primitive types, we currently support PyArrow, nanoarrow,
- and arro3 objects in addition to primitive data types. Other objects
- that support the Arrow FFI standard will be given a "best attempt" at
- conversion to scalar objects.
-
-
-
- .. py:method:: update(*values: pyarrow.Array) -> None
- :abstractmethod:
-
-
- Evaluate an array of values and update state.
-
-
-
-.. py:class:: AggregateUDF(name: str, accumulator: collections.abc.Callable[[], Accumulator], input_types: list[pyarrow.DataType], return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str)
- AggregateUDF(name: str, accumulator: AggregateUDFExportable, input_types: None = ..., return_type: None = ..., state_type: None = ..., volatility: None = ...)
-
- Class for performing scalar user-defined functions (UDF).
-
- Aggregate UDFs operate on a group of rows and return a single value. See
- also :py:class:`ScalarUDF` for operating on a row by row basis.
-
- Instantiate a user-defined aggregate function (UDAF).
-
- See :py:func:`udaf` for a convenience function and argument
- descriptions.
-
-
- .. py:method:: __call__(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
-
- Execute the UDAF.
-
- This function is not typically called by an end user. These calls will
- occur during the evaluation of the dataframe.
-
-
-
- .. py:method:: __repr__() -> str
-
- Print a string representation of the Aggregate UDF.
-
-
-
- .. py:method:: from_pycapsule(func: AggregateUDFExportable | _typeshed.CapsuleType) -> AggregateUDF
- :staticmethod:
-
-
- Create an Aggregate UDF from AggregateUDF PyCapsule object.
-
- This function will instantiate a Aggregate UDF that uses a DataFusion
- AggregateUDF that is exported via the FFI bindings.
-
-
-
- .. py:method:: udaf(input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str, name: str | None = None) -> collections.abc.Callable[Ellipsis, AggregateUDF]
- udaf(accum: collections.abc.Callable[[], Accumulator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str, name: str | None = None) -> AggregateUDF
- udaf(accum: AggregateUDFExportable) -> AggregateUDF
- udaf(accum: _typeshed.CapsuleType) -> AggregateUDF
- :staticmethod:
-
-
- Create a new User-Defined Aggregate Function (UDAF).
-
- This class allows you to define an aggregate function that can be used in
- data aggregation or window function calls.
-
- Usage:
- - As a function: ``udaf(accum, input_types, return_type, state_type, volatility, name)``.
- - As a decorator: ``@udaf(input_types, return_type, state_type, volatility, name)``.
- When using ``udaf`` as a decorator, do not pass ``accum`` explicitly.
-
- Function example:
-
- If your :py:class:`Accumulator` can be instantiated with no arguments, you
- can simply pass it's type as `accum`. If you need to pass additional
- arguments to it's constructor, you can define a lambda or a factory method.
- During runtime the :py:class:`Accumulator` will be constructed for every
- instance in which this UDAF is used. The following examples are all valid::
-
- import pyarrow as pa
- import pyarrow.compute as pc
-
- class Summarize(Accumulator):
- def __init__(self, bias: float = 0.0):
- self._sum = pa.scalar(bias)
-
- def state(self) -> list[pa.Scalar]:
- return [self._sum]
-
- def update(self, values: pa.Array) -> None:
- self._sum = pa.scalar(self._sum.as_py() + pc.sum(values).as_py())
-
- def merge(self, states: list[pa.Array]) -> None:
- self._sum = pa.scalar(self._sum.as_py() + pc.sum(states[0]).as_py())
-
- def evaluate(self) -> pa.Scalar:
- return self._sum
-
- def sum_bias_10() -> Summarize:
- return Summarize(10.0)
-
- udaf1 = udaf(Summarize, pa.float64(), pa.float64(), [pa.float64()],
- "immutable")
- udaf2 = udaf(sum_bias_10, pa.float64(), pa.float64(), [pa.float64()],
- "immutable")
- udaf3 = udaf(lambda: Summarize(20.0), pa.float64(), pa.float64(),
- [pa.float64()], "immutable")
-
- Decorator example:::
-
- @udaf(pa.float64(), pa.float64(), [pa.float64()], "immutable")
- def udf4() -> Summarize:
- return Summarize(10.0)
-
- :param accum: The accumulator python function. Only needed when calling as a
- function. Skip this argument when using ``udaf`` as a decorator.
- If you have a Rust backed AggregateUDF within a PyCapsule, you can
- pass this parameter and ignore the rest. They will be determined
- directly from the underlying function. See the online documentation
- for more information.
- :param input_types: The data types of the arguments to ``accum``.
- :param return_type: The data type of the return value.
- :param state_type: The data types of the intermediate accumulation.
- :param volatility: See :py:class:`Volatility` for allowed values.
- :param name: A descriptive name for the function.
-
- :returns: A user-defined aggregate function, which can be used in either data
- aggregation or window function calls.
-
-
-
- .. py:attribute:: _udaf
-
-
.. py:class:: Catalog(catalog: datafusion._internal.catalog.RawCatalog)
DataFusion data catalog.
@@ -282,12 +99,6 @@ Package Contents
- .. py:method:: database(name: str = 'public') -> Schema
-
- Returns the database with the given ``name`` from this catalog.
-
-
-
.. py:method:: deregister_schema(name: str, cascade: bool = True) -> Schema | None
Deregister a schema from this catalog.
@@ -562,16 +373,6 @@ Package Contents
.. py:attribute:: _raw_write_options
-.. py:class:: Database(schema: datafusion._internal.catalog.RawSchema)
-
- Bases: :py:obj:`Schema`
-
-
- See `Schema`.
-
- This constructor is not typically called by the end user.
-
-
.. py:class:: ExecutionPlan(plan: datafusion._internal.ExecutionPlan)
Represent nodes in the DataFusion Physical Plan.
@@ -634,6 +435,44 @@ Package Contents
Returns the number of partitions in the physical plan.
+.. py:class:: ExplainFormat
+
+ Bases: :py:obj:`enum.Enum`
+
+
+ Output format for explain plans.
+
+ Controls how the query plan is rendered in :py:meth:`DataFrame.explain`.
+
+
+ .. py:attribute:: GRAPHVIZ
+ :value: 'graphviz'
+
+
+ Graphviz DOT format for graph rendering.
+
+
+ .. py:attribute:: INDENT
+ :value: 'indent'
+
+
+ Default indented text format.
+
+
+ .. py:attribute:: PGJSON
+ :value: 'pgjson'
+
+
+ PostgreSQL-compatible JSON format for use with visualization tools.
+
+
+ .. py:attribute:: TREE
+ :value: 'tree'
+
+
+ Tree-style visual format with box-drawing characters.
+
+
.. py:class:: Expr(expr: datafusion._internal.expr.RawExpr)
Expression object.
@@ -1002,14 +841,6 @@ Package Contents
- .. py:method:: display_name() -> str
-
- Returns the name of this expression as it should appear in a schema.
-
- This name will not include any CAST expressions.
-
-
-
.. py:method:: distinct() -> ExprFuncBuilder
Only evaluate distinct values for an aggregate function.
@@ -1974,9 +1805,9 @@ Package Contents
:returns: A new :py:class:`RuntimeEnvBuilder` object with the updated setting.
- Examples usage::
+ .. rubric:: Examples
- config = RuntimeEnvBuilder().with_fair_spill_pool(1024)
+ >>> config = dfn.RuntimeEnvBuilder().with_fair_spill_pool(1024)
@@ -1992,9 +1823,9 @@ Package Contents
:returns: A new :py:class:`RuntimeEnvBuilder` object with the updated setting.
- Example usage::
+ .. rubric:: Examples
- config = RuntimeEnvBuilder().with_greedy_memory_pool(1024)
+ >>> config = dfn.RuntimeEnvBuilder().with_greedy_memory_pool(1024)
@@ -2006,9 +1837,9 @@ Package Contents
:returns: A new :py:class:`RuntimeEnvBuilder` object with the updated setting.
- Example usage::
+ .. rubric:: Examples
- config = RuntimeEnvBuilder().with_temp_file_path("/tmp")
+ >>> config = dfn.RuntimeEnvBuilder().with_temp_file_path("/tmp")
@@ -2045,9 +1876,9 @@ Package Contents
:returns: A new :py:class:`SQLOptions` object with the updated setting.
- Example usage::
+ .. rubric:: Examples
- options = SQLOptions().with_allow_ddl(True)
+ >>> options = dfn.SQLOptions().with_allow_ddl(True)
@@ -2061,9 +1892,9 @@ Package Contents
:returns: A new :py:class:`SQLOptions` object with the updated setting.
- Example usage::
+ .. rubric:: Examples
- options = SQLOptions().with_allow_dml(True)
+ >>> options = dfn.SQLOptions().with_allow_dml(True)
@@ -2076,110 +1907,15 @@ Package Contents
:returns: py:class:SQLOptions` object with the updated setting.
:rtype: A new
- Example usage::
+ .. rubric:: Examples
- options = SQLOptions().with_allow_statements(True)
+ >>> options = dfn.SQLOptions().with_allow_statements(True)
.. py:attribute:: options_internal
-.. py:class:: ScalarUDF(name: str, func: collections.abc.Callable[Ellipsis, _R], input_fields: list[pyarrow.Field], return_field: _R, volatility: Volatility | str)
-
- Class for performing scalar user-defined functions (UDF).
-
- Scalar UDFs operate on a row by row basis. See also :py:class:`AggregateUDF` for
- operating on a group of rows.
-
- Instantiate a scalar user-defined function (UDF).
-
- See helper method :py:func:`udf` for argument details.
-
-
- .. py:method:: __call__(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
-
- Execute the UDF.
-
- This function is not typically called by an end user. These calls will
- occur during the evaluation of the dataframe.
-
-
-
- .. py:method:: __repr__() -> str
-
- Print a string representation of the Scalar UDF.
-
-
-
- .. py:method:: from_pycapsule(func: ScalarUDFExportable) -> ScalarUDF
- :staticmethod:
-
-
- Create a Scalar UDF from ScalarUDF PyCapsule object.
-
- This function will instantiate a Scalar UDF that uses a DataFusion
- ScalarUDF that is exported via the FFI bindings.
-
-
-
- .. py:method:: udf(input_fields: collections.abc.Sequence[pyarrow.DataType | pyarrow.Field] | pyarrow.DataType | pyarrow.Field, return_field: pyarrow.DataType | pyarrow.Field, volatility: Volatility | str, name: str | None = None) -> collections.abc.Callable[Ellipsis, ScalarUDF]
- udf(func: collections.abc.Callable[Ellipsis, _R], input_fields: collections.abc.Sequence[pyarrow.DataType | pyarrow.Field] | pyarrow.DataType | pyarrow.Field, return_field: pyarrow.DataType | pyarrow.Field, volatility: Volatility | str, name: str | None = None) -> ScalarUDF
- udf(func: ScalarUDFExportable) -> ScalarUDF
- :staticmethod:
-
-
- Create a new User-Defined Function (UDF).
-
- This class can be used both as either a function or a decorator.
-
- Usage:
- - As a function: ``udf(func, input_fields, return_field, volatility, name)``.
- - As a decorator: ``@udf(input_fields, return_field, volatility, name)``.
- When used a decorator, do **not** pass ``func`` explicitly.
-
- In lieu of passing a PyArrow Field, you can pass a DataType for simplicity.
- When you do so, it will be assumed that the nullability of the inputs and
- output are True and that they have no metadata.
-
- :param func: Only needed when calling as a function.
- Skip this argument when using `udf` as a decorator. If you have a Rust
- backed ScalarUDF within a PyCapsule, you can pass this parameter
- and ignore the rest. They will be determined directly from the
- underlying function. See the online documentation for more information.
- :type func: Callable, optional
- :param input_fields: The data types or Fields
- of the arguments to ``func``. This list must be of the same length
- as the number of arguments.
- :type input_fields: list[pa.Field | pa.DataType]
- :param return_field: The field of the return value from the function.
- :type return_field: _R
- :param volatility: See `Volatility` for allowed values.
- :type volatility: Volatility | str
- :param name: A descriptive name for the function.
- :type name: Optional[str]
-
- :returns: A user-defined function that can be used in SQL expressions,
- data aggregation, or window function calls.
-
- Example: Using ``udf`` as a function::
-
- def double_func(x):
- return x * 2
- double_udf = udf(double_func, [pa.int32()], pa.int32(),
- "volatile", "double_it")
-
- Example: Using ``udf`` as a decorator::
-
- @udf([pa.int32()], pa.int32(), "volatile", "double_it")
- def double_udf(x):
- return x * 2
-
-
-
- .. py:attribute:: _udf
-
-
.. py:class:: SessionConfig(config_options: dict[str, str] | None = None)
Session configuration options.
@@ -2233,6 +1969,17 @@ Package Contents
+ .. py:method:: with_extension(extension: Any) -> SessionConfig
+
+ Create a new configuration using an extension.
+
+ :param extension: A custom configuration extension object. These are
+ :param shared from another DataFusion extension library.:
+
+ :returns: A new :py:class:`SessionConfig` object with the updated setting.
+
+
+
.. py:method:: with_information_schema(enabled: bool = True) -> SessionConfig
Enable or disable the inclusion of ``information_schema`` virtual tables.
@@ -2388,59 +2135,6 @@ Package Contents
Returns the schema associated with this table.
-.. py:class:: TableFunction(name: str, func: collections.abc.Callable[[], any], ctx: datafusion.SessionContext | None = None)
-
- Class for performing user-defined table functions (UDTF).
-
- Table functions generate new table providers based on the
- input expressions.
-
- Instantiate a user-defined table function (UDTF).
-
- See :py:func:`udtf` for a convenience function and argument
- descriptions.
-
-
- .. py:method:: __call__(*args: datafusion.expr.Expr) -> Any
-
- Execute the UDTF and return a table provider.
-
-
-
- .. py:method:: __repr__() -> str
-
- User printable representation.
-
-
-
- .. py:method:: _create_table_udf(func: collections.abc.Callable[Ellipsis, Any], name: str) -> TableFunction
- :staticmethod:
-
-
- Create a TableFunction instance from function arguments.
-
-
-
- .. py:method:: _create_table_udf_decorator(name: str | None = None) -> collections.abc.Callable[[collections.abc.Callable[[], WindowEvaluator]], collections.abc.Callable[Ellipsis, datafusion.expr.Expr]]
- :staticmethod:
-
-
- Create a decorator for a WindowUDF.
-
-
-
- .. py:method:: udtf(name: str) -> collections.abc.Callable[Ellipsis, Any]
- udtf(func: collections.abc.Callable[[], Any], name: str) -> TableFunction
- :staticmethod:
-
-
- Create a new User-Defined Table Function (UDTF).
-
-
-
- .. py:attribute:: _udtf
-
-
.. py:class:: WindowFrame(units: str, start_bound: Any | None, end_bound: Any | None)
Defines a window frame for performing window operations.
@@ -2483,134 +2177,6 @@ Package Contents
.. py:attribute:: window_frame
-.. py:class:: WindowUDF(name: str, func: collections.abc.Callable[[], WindowEvaluator], input_types: list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str)
-
- Class for performing window user-defined functions (UDF).
-
- Window UDFs operate on a partition of rows. See
- also :py:class:`ScalarUDF` for operating on a row by row basis.
-
- Instantiate a user-defined window function (UDWF).
-
- See :py:func:`udwf` for a convenience function and argument
- descriptions.
-
-
- .. py:method:: __call__(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
-
- Execute the UDWF.
-
- This function is not typically called by an end user. These calls will
- occur during the evaluation of the dataframe.
-
-
-
- .. py:method:: __repr__() -> str
-
- Print a string representation of the Window UDF.
-
-
-
- .. py:method:: _create_window_udf(func: collections.abc.Callable[[], WindowEvaluator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: str | None = None) -> WindowUDF
- :staticmethod:
-
-
- Create a WindowUDF instance from function arguments.
-
-
-
- .. py:method:: _create_window_udf_decorator(input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: str | None = None) -> collections.abc.Callable[[collections.abc.Callable[[], WindowEvaluator]], collections.abc.Callable[Ellipsis, datafusion.expr.Expr]]
- :staticmethod:
-
-
- Create a decorator for a WindowUDF.
-
-
-
- .. py:method:: _get_default_name(func: collections.abc.Callable) -> str
- :staticmethod:
-
-
- Get the default name for a function based on its attributes.
-
-
-
- .. py:method:: _normalize_input_types(input_types: pyarrow.DataType | list[pyarrow.DataType]) -> list[pyarrow.DataType]
- :staticmethod:
-
-
- Convert a single DataType to a list if needed.
-
-
-
- .. py:method:: from_pycapsule(func: WindowUDFExportable) -> WindowUDF
- :staticmethod:
-
-
- Create a Window UDF from WindowUDF PyCapsule object.
-
- This function will instantiate a Window UDF that uses a DataFusion
- WindowUDF that is exported via the FFI bindings.
-
-
-
- .. py:method:: udwf(input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: str | None = None) -> collections.abc.Callable[Ellipsis, WindowUDF]
- udwf(func: collections.abc.Callable[[], WindowEvaluator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: str | None = None) -> WindowUDF
- :staticmethod:
-
-
- Create a new User-Defined Window Function (UDWF).
-
- This class can be used both as either a function or a decorator.
-
- Usage:
- - As a function: ``udwf(func, input_types, return_type, volatility, name)``.
- - As a decorator: ``@udwf(input_types, return_type, volatility, name)``.
- When using ``udwf`` as a decorator, do not pass ``func`` explicitly.
-
- Function example::
-
- import pyarrow as pa
-
- class BiasedNumbers(WindowEvaluator):
- def __init__(self, start: int = 0) -> None:
- self.start = start
-
- def evaluate_all(self, values: list[pa.Array],
- num_rows: int) -> pa.Array:
- return pa.array([self.start + i for i in range(num_rows)])
-
- def bias_10() -> BiasedNumbers:
- return BiasedNumbers(10)
-
- udwf1 = udwf(BiasedNumbers, pa.int64(), pa.int64(), "immutable")
- udwf2 = udwf(bias_10, pa.int64(), pa.int64(), "immutable")
- udwf3 = udwf(lambda: BiasedNumbers(20), pa.int64(), pa.int64(), "immutable")
-
-
- Decorator example::
-
- @udwf(pa.int64(), pa.int64(), "immutable")
- def biased_numbers() -> BiasedNumbers:
- return BiasedNumbers(10)
-
- :param func: Only needed when calling as a function. Skip this argument when
- using ``udwf`` as a decorator. If you have a Rust backed WindowUDF
- within a PyCapsule, you can pass this parameter and ignore the rest.
- They will be determined directly from the underlying function. See
- the online documentation for more information.
- :param input_types: The data types of the arguments.
- :param return_type: The data type of the return value.
- :param volatility: See :py:class:`Volatility` for allowed values.
- :param name: A descriptive name for the function.
-
- :returns: A user-defined window function that can be used in window function calls.
-
-
-
- .. py:attribute:: _udwf
-
-
.. py:function:: configure_formatter(**kwargs: Any) -> None
Configure the global DataFrame HTML formatter.
@@ -2625,7 +2191,7 @@ Package Contents
.. rubric:: Example
- >>> from datafusion.html_formatter import configure_formatter
+ >>> from datafusion.dataframe_formatter import configure_formatter
>>> configure_formatter(
... max_cell_length=50,
... max_height=500,
@@ -2740,11 +2306,3 @@ Package Contents
.. py:data:: column
:type: Col
-.. py:data:: udaf
-
-.. py:data:: udf
-
-.. py:data:: udtf
-
-.. py:data:: udwf
-
diff --git a/_sources/autoapi/datafusion/user_defined/index.rst.txt b/_sources/autoapi/datafusion/user_defined/index.rst.txt
index 21063829..80690d67 100644
--- a/_sources/autoapi/datafusion/user_defined/index.rst.txt
+++ b/_sources/autoapi/datafusion/user_defined/index.rst.txt
@@ -161,48 +161,72 @@ Module Contents
- As a decorator: ``@udaf(input_types, return_type, state_type, volatility, name)``.
When using ``udaf`` as a decorator, do not pass ``accum`` explicitly.
- Function example:
-
If your :py:class:`Accumulator` can be instantiated with no arguments, you
- can simply pass it's type as `accum`. If you need to pass additional
- arguments to it's constructor, you can define a lambda or a factory method.
+ can simply pass its type as ``accum``. If you need to pass additional
+ arguments to its constructor, you can define a lambda or a factory method.
During runtime the :py:class:`Accumulator` will be constructed for every
- instance in which this UDAF is used. The following examples are all valid::
-
- import pyarrow as pa
- import pyarrow.compute as pc
-
- class Summarize(Accumulator):
- def __init__(self, bias: float = 0.0):
- self._sum = pa.scalar(bias)
-
- def state(self) -> list[pa.Scalar]:
- return [self._sum]
-
- def update(self, values: pa.Array) -> None:
- self._sum = pa.scalar(self._sum.as_py() + pc.sum(values).as_py())
-
- def merge(self, states: list[pa.Array]) -> None:
- self._sum = pa.scalar(self._sum.as_py() + pc.sum(states[0]).as_py())
-
- def evaluate(self) -> pa.Scalar:
- return self._sum
-
- def sum_bias_10() -> Summarize:
- return Summarize(10.0)
-
- udaf1 = udaf(Summarize, pa.float64(), pa.float64(), [pa.float64()],
- "immutable")
- udaf2 = udaf(sum_bias_10, pa.float64(), pa.float64(), [pa.float64()],
- "immutable")
- udaf3 = udaf(lambda: Summarize(20.0), pa.float64(), pa.float64(),
- [pa.float64()], "immutable")
-
- Decorator example:::
-
- @udaf(pa.float64(), pa.float64(), [pa.float64()], "immutable")
- def udf4() -> Summarize:
- return Summarize(10.0)
+ instance in which this UDAF is used.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow.compute as pc
+ >>> from datafusion.user_defined import AggregateUDF, Accumulator, udaf
+ >>> class Summarize(Accumulator):
+ ... def __init__(self, bias: float = 0.0):
+ ... self._sum = pa.scalar(bias)
+ ... def state(self):
+ ... return [self._sum]
+ ... def update(self, values):
+ ... self._sum = pa.scalar(
+ ... self._sum.as_py() + pc.sum(values).as_py())
+ ... def merge(self, states):
+ ... self._sum = pa.scalar(
+ ... self._sum.as_py() + pc.sum(states[0]).as_py())
+ ... def evaluate(self):
+ ... return self._sum
+
+ Using ``udaf`` as a function:
+
+ >>> udaf1 = AggregateUDF.udaf(
+ ... Summarize, pa.float64(), pa.float64(),
+ ... [pa.float64()], "immutable")
+
+ Wrapping ``udaf`` with a function:
+
+ >>> def sum_bias_10() -> Summarize:
+ ... return Summarize(10.0)
+ >>> udaf2 = udaf(sum_bias_10, pa.float64(), pa.float64(), [pa.float64()],
+ ... "immutable")
+
+ Using ``udaf`` with lambda:
+
+ >>> udaf3 = udaf(lambda: Summarize(20.0), pa.float64(), pa.float64(),
+ ... [pa.float64()], "immutable")
+
+ Using ``udaf`` as a decorator:
+
+ >>> @AggregateUDF.udaf(
+ ... pa.float64(), pa.float64(),
+ ... [pa.float64()], "immutable")
+ ... def udaf4():
+ ... return Summarize(10.0)
+
+ Apply to a dataframe:
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0]})
+ >>> df.aggregate([], [udaf1(col("a")).alias("total")]).collect_column(
+ ... "total")[0].as_py()
+ 6.0
+ >>> df.aggregate([], [udaf2(col("a")).alias("total")]).collect_column(
+ ... "total")[0].as_py()
+ 16.0
+ >>> df.aggregate([], [udaf3(col("a")).alias("total")]).collect_column(
+ ... "total")[0].as_py()
+ 26.0
+ >>> df.aggregate([], [udaf4(col("a")).alias("total")]).collect_column(
+ ... "total")[0].as_py()
+ 16.0
:param accum: The accumulator python function. Only needed when calling as a
function. Skip this argument when using ``udaf`` as a decorator.
@@ -284,7 +308,8 @@ Module Contents
This class can be used both as either a function or a decorator.
Usage:
- - As a function: ``udf(func, input_fields, return_field, volatility, name)``.
+ - As a function: ``udf(func, input_fields, return_field,
+ volatility, name)``.
- As a decorator: ``@udf(input_fields, return_field, volatility, name)``.
When used a decorator, do **not** pass ``func`` explicitly.
@@ -312,18 +337,32 @@ Module Contents
:returns: A user-defined function that can be used in SQL expressions,
data aggregation, or window function calls.
- Example: Using ``udf`` as a function::
+ .. rubric:: Examples
- def double_func(x):
- return x * 2
- double_udf = udf(double_func, [pa.int32()], pa.int32(),
- "volatile", "double_it")
+ Using ``udf`` as a function:
- Example: Using ``udf`` as a decorator::
+ >>> import pyarrow.compute as pc
+ >>> from datafusion.user_defined import ScalarUDF
+ >>> def double_func(x):
+ ... return pc.multiply(x, 2)
+ >>> double_udf = ScalarUDF.udf(
+ ... double_func, [pa.int64()], pa.int64(),
+ ... "volatile", "double_it")
- @udf([pa.int32()], pa.int32(), "volatile", "double_it")
- def double_udf(x):
- return x * 2
+ Using ``udf`` as a decorator:
+
+ >>> @ScalarUDF.udf([pa.int64()], pa.int64(), "volatile")
+ ... def decorator_double_udf(x):
+ ... return pc.multiply(x, 3)
+
+ Apply to a dataframe:
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1, 2, 3]})
+ >>> df.select(double_udf(col("x")).alias("result")).to_pydict()
+ {'result': [2, 4, 6]}
+ >>> df.select(decorator_double_udf(col("x")).alias("result")).to_pydict()
+ {'result': [3, 6, 9]}
@@ -707,31 +746,45 @@ Module Contents
- As a decorator: ``@udwf(input_types, return_type, volatility, name)``.
When using ``udwf`` as a decorator, do not pass ``func`` explicitly.
- Function example::
-
- import pyarrow as pa
-
- class BiasedNumbers(WindowEvaluator):
- def __init__(self, start: int = 0) -> None:
- self.start = start
-
- def evaluate_all(self, values: list[pa.Array],
- num_rows: int) -> pa.Array:
- return pa.array([self.start + i for i in range(num_rows)])
-
- def bias_10() -> BiasedNumbers:
- return BiasedNumbers(10)
-
- udwf1 = udwf(BiasedNumbers, pa.int64(), pa.int64(), "immutable")
- udwf2 = udwf(bias_10, pa.int64(), pa.int64(), "immutable")
- udwf3 = udwf(lambda: BiasedNumbers(20), pa.int64(), pa.int64(), "immutable")
-
-
- Decorator example::
-
- @udwf(pa.int64(), pa.int64(), "immutable")
- def biased_numbers() -> BiasedNumbers:
- return BiasedNumbers(10)
+ .. rubric:: Examples
+
+ >>> from datafusion.user_defined import WindowUDF, WindowEvaluator, udwf
+ >>> class BiasedNumbers(WindowEvaluator):
+ ... def __init__(self, start: int = 0):
+ ... self.start = start
+ ... def evaluate_all(self, values, num_rows):
+ ... return pa.array(
+ ... [self.start + i for i in range(num_rows)])
+
+ Using ``udwf`` as a function:
+
+ >>> udwf1 = WindowUDF.udwf(
+ ... BiasedNumbers, pa.int64(), pa.int64(), "immutable")
+ >>> def bias_10() -> BiasedNumbers:
+ ... return BiasedNumbers(10)
+ >>> udwf2 = udwf(bias_10, pa.int64(), pa.int64(), "immutable")
+ >>> udwf3 = udwf(
+ ... lambda: BiasedNumbers(20), pa.int64(), pa.int64(), "immutable"
+ ... )
+
+ Using ``udwf`` as a decorator:
+
+ >>> @WindowUDF.udwf(pa.int64(), pa.int64(), "immutable")
+ ... def biased_numbers():
+ ... return BiasedNumbers(10)
+
+ Apply to a dataframe:
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [10, 20, 30]})
+ >>> df.select(udwf1(col("a")).alias("result")).to_pydict()
+ {'result': [0, 1, 2]}
+ >>> df.select(udwf2(col("a")).alias("result")).to_pydict()
+ {'result': [10, 11, 12]}
+ >>> df.select(udwf3(col("a")).alias("result")).to_pydict()
+ {'result': [20, 21, 22]}
+ >>> df.select(biased_numbers(col("a")).alias("result")).to_pydict()
+ {'result': [10, 11, 12]}
:param func: Only needed when calling as a function. Skip this argument when
using ``udwf`` as a decorator. If you have a Rust backed WindowUDF
diff --git a/_sources/contributor-guide/ffi.rst.txt b/_sources/contributor-guide/ffi.rst.txt
index 5006b0ca..e0158e0a 100644
--- a/_sources/contributor-guide/ffi.rst.txt
+++ b/_sources/contributor-guide/ffi.rst.txt
@@ -156,7 +156,7 @@ instead of mutating the container directly:
.. code-block:: rust
- #[pyclass(name = "Config", module = "datafusion", subclass, frozen)]
+ #[pyclass(from_py_object, name = "Config", module = "datafusion", subclass, frozen)]
#[derive(Clone)]
pub(crate) struct PyConfig {
config: Arc<RwLock<ConfigOptions>>,
@@ -170,7 +170,7 @@ existing instance in place:
.. code-block:: rust
- #[pyclass(frozen, name = "SessionContext", module = "datafusion", subclass)]
+ #[pyclass(from_py_object, frozen, name = "SessionContext", module = "datafusion", subclass)]
#[derive(Clone)]
pub struct PySessionContext {
pub ctx: SessionContext,
@@ -186,7 +186,7 @@ field updates:
// TODO: This looks like this needs pyo3 tracking so leaving unfrozen for now
#[derive(Debug, Clone)]
- #[pyclass(name = "DataTypeMap", module = "datafusion.common", subclass)]
+ #[pyclass(from_py_object, name = "DataTypeMap", module = "datafusion.common", subclass)]
pub struct DataTypeMap {
#[pyo3(get, set)]
pub arrow_type: PyDataType,
@@ -232,8 +232,11 @@ can then be turned into a ``ForeignTableProvider`` the associated code is:
.. code-block:: rust
- let capsule = capsule.downcast::<PyCapsule>()?;
- let provider = unsafe { capsule.reference::<FFI_TableProvider>() };
+ let capsule = capsule.cast::<PyCapsule>()?;
+ let data: NonNull<FFI_TableProvider> = capsule
+ .pointer_checked(Some(name))?
+ .cast();
+ let codec = unsafe { data.as_ref() };
By convention the ``datafusion-python`` library expects a Python object that has a
``TableProvider`` PyCapsule to have this capsule accessible by calling a function named
diff --git a/_sources/user-guide/common-operations/aggregations.rst.txt b/_sources/user-guide/common-operations/aggregations.rst.txt
index e458e5fc..de24a2ba 100644
--- a/_sources/user-guide/common-operations/aggregations.rst.txt
+++ b/_sources/user-guide/common-operations/aggregations.rst.txt
@@ -163,6 +163,168 @@ Suppose we want to find the speed values for only Pokemon that have low Attack v
f.avg(col_speed, filter=col_attack < lit(50)).alias("Avg Speed Low Attack")])
+Grouping Sets
+-------------
+
+The default style of aggregation produces one row per group. Sometimes you want a single query to
+produce rows at multiple levels of detail — for example, totals per type *and* an overall grand
+total, or subtotals for every combination of two columns plus the individual column totals. Writing
+separate queries and concatenating them is tedious and runs the data multiple times. Grouping sets
+solve this by letting you specify several grouping levels in one pass.
+
+DataFusion supports three grouping set styles through the
+:py:class:`~datafusion.expr.GroupingSet` class:
+
+- :py:meth:`~datafusion.expr.GroupingSet.rollup` — hierarchical subtotals, like a drill-down report
+- :py:meth:`~datafusion.expr.GroupingSet.cube` — every possible subtotal combination, like a pivot table
+- :py:meth:`~datafusion.expr.GroupingSet.grouping_sets` — explicitly list exactly which grouping levels you want
+
+Because result rows come from different grouping levels, a column that is *not* part of a
+particular level will be ``null`` in that row. Use :py:func:`~datafusion.functions.grouping` to
+distinguish a real ``null`` in the data from one that means "this column was aggregated across."
+It returns ``0`` when the column is a grouping key for that row, and ``1`` when it is not.
+
+Rollup
+^^^^^^
+
+:py:meth:`~datafusion.expr.GroupingSet.rollup` creates a hierarchy. ``rollup(a, b)`` produces
+grouping sets ``(a, b)``, ``(a)``, and ``()`` — like nested subtotals in a report. This is useful
+when your columns have a natural hierarchy, such as region → city or type → subtype.
+
+Suppose we want to summarize Pokemon stats by ``Type 1`` with subtotals and a grand total. With
+the default aggregation style we would need two separate queries. With ``rollup`` we get it all at
+once:
+
+.. ipython:: python
+
+ from datafusion.expr import GroupingSet
+
+ df.aggregate(
+ [GroupingSet.rollup(col_type_1)],
+ [f.count(col_speed).alias("Count"),
+ f.avg(col_speed).alias("Avg Speed"),
+ f.max(col_speed).alias("Max Speed")]
+ ).sort(col_type_1.sort(ascending=True, nulls_first=True))
+
+The first row — where ``Type 1`` is ``null`` — is the grand total across all types. But how do you
+tell a grand-total ``null`` apart from a Pokemon that genuinely has no type? The
+:py:func:`~datafusion.functions.grouping` function returns ``0`` when the column is a grouping key
+for that row and ``1`` when it is aggregated across.
+
+.. note::
+
+ Due to an upstream DataFusion limitation
+ (`apache/datafusion#21411 <https://github.com/apache/datafusion/issues/21411>`_),
+ ``.alias()`` cannot be applied directly to a ``grouping()`` expression — it will raise an
+ error at execution time. Instead, use
+ :py:meth:`~datafusion.dataframe.DataFrame.with_column_renamed` on the result DataFrame to
+ give the column a readable name. Once the upstream issue is resolved, you will be able to
+ use ``.alias()`` directly and the workaround below will no longer be necessary.
+
+The raw column name generated by ``grouping()`` contains internal identifiers, so we use
+:py:meth:`~datafusion.dataframe.DataFrame.with_column_renamed` to clean it up:
+
+.. ipython:: python
+
+ result = df.aggregate(
+ [GroupingSet.rollup(col_type_1)],
+ [f.count(col_speed).alias("Count"),
+ f.avg(col_speed).alias("Avg Speed"),
+ f.grouping(col_type_1)]
+ )
+ for field in result.schema():
+ if field.name.startswith("grouping("):
+ result = result.with_column_renamed(field.name, "Is Total")
+ result.sort(col_type_1.sort(ascending=True, nulls_first=True))
+
+With two columns the hierarchy becomes more apparent. ``rollup(Type 1, Type 2)`` produces:
+
+- one row per ``(Type 1, Type 2)`` pair — the most detailed level
+- one row per ``Type 1`` — subtotals
+- one grand total row
+
+.. ipython:: python
+
+ df.aggregate(
+ [GroupingSet.rollup(col_type_1, col_type_2)],
+ [f.count(col_speed).alias("Count"),
+ f.avg(col_speed).alias("Avg Speed")]
+ ).sort(
+ col_type_1.sort(ascending=True, nulls_first=True),
+ col_type_2.sort(ascending=True, nulls_first=True)
+ )
+
+Cube
+^^^^
+
+:py:meth:`~datafusion.expr.GroupingSet.cube` produces every possible subset. ``cube(a, b)``
+produces grouping sets ``(a, b)``, ``(a)``, ``(b)``, and ``()`` — one more than ``rollup`` because
+it also includes ``(b)`` alone. This is useful when neither column is "above" the other in a
+hierarchy and you want all cross-tabulations.
+
+For our Pokemon data, ``cube(Type 1, Type 2)`` gives us stats broken down by the type pair,
+by ``Type 1`` alone, by ``Type 2`` alone, and a grand total — all in one query:
+
+.. ipython:: python
+
+ df.aggregate(
+ [GroupingSet.cube(col_type_1, col_type_2)],
+ [f.count(col_speed).alias("Count"),
+ f.avg(col_speed).alias("Avg Speed")]
+ ).sort(
+ col_type_1.sort(ascending=True, nulls_first=True),
+ col_type_2.sort(ascending=True, nulls_first=True)
+ )
+
+Compared to the ``rollup`` example above, notice the extra rows where ``Type 1`` is ``null`` but
+``Type 2`` has a value — those are the per-``Type 2`` subtotals that ``rollup`` does not include.
+
+Explicit Grouping Sets
+^^^^^^^^^^^^^^^^^^^^^^
+
+:py:meth:`~datafusion.expr.GroupingSet.grouping_sets` lets you list exactly which grouping levels
+you need when ``rollup`` or ``cube`` would produce too many or too few. Each argument is a list of
+columns forming one grouping set.
+
+For example, if we want only the per-``Type 1`` totals and per-``Type 2`` totals — but *not* the
+full ``(Type 1, Type 2)`` detail rows or the grand total — we can ask for exactly that:
+
+.. ipython:: python
+
+ df.aggregate(
+ [GroupingSet.grouping_sets([col_type_1], [col_type_2])],
+ [f.count(col_speed).alias("Count"),
+ f.avg(col_speed).alias("Avg Speed")]
+ ).sort(
+ col_type_1.sort(ascending=True, nulls_first=True),
+ col_type_2.sort(ascending=True, nulls_first=True)
+ )
+
+Each row belongs to exactly one grouping level. The :py:func:`~datafusion.functions.grouping`
+function tells you which level each row comes from:
+
+.. ipython:: python
+
+ result = df.aggregate(
+ [GroupingSet.grouping_sets([col_type_1], [col_type_2])],
+ [f.count(col_speed).alias("Count"),
+ f.avg(col_speed).alias("Avg Speed"),
+ f.grouping(col_type_1),
+ f.grouping(col_type_2)]
+ )
+ for field in result.schema():
+ if field.name.startswith("grouping("):
+ clean = field.name.split(".")[-1].rstrip(")")
+ result = result.with_column_renamed(field.name, f"grouping({clean})")
+ result.sort(
+ col_type_1.sort(ascending=True, nulls_first=True),
+ col_type_2.sort(ascending=True, nulls_first=True)
+ )
+
+Where ``grouping(Type 1)`` is ``0`` the row is a per-``Type 1`` total (and ``Type 2`` is ``null``).
+Where ``grouping(Type 2)`` is ``0`` the row is a per-``Type 2`` total (and ``Type 1`` is ``null``).
+
+
Aggregate Functions
-------------------
@@ -192,6 +354,7 @@ The available aggregate functions are:
- :py:func:`datafusion.functions.stddev_pop`
- :py:func:`datafusion.functions.var_samp`
- :py:func:`datafusion.functions.var_pop`
+ - :py:func:`datafusion.functions.var_population`
6. Linear Regression Functions
- :py:func:`datafusion.functions.regr_count`
- :py:func:`datafusion.functions.regr_slope`
@@ -208,9 +371,16 @@ The available aggregate functions are:
- :py:func:`datafusion.functions.nth_value`
8. String Functions
- :py:func:`datafusion.functions.string_agg`
-9. Approximation Functions
+9. Percentile Functions
+ - :py:func:`datafusion.functions.percentile_cont`
+ - :py:func:`datafusion.functions.quantile_cont`
- :py:func:`datafusion.functions.approx_distinct`
- :py:func:`datafusion.functions.approx_median`
- :py:func:`datafusion.functions.approx_percentile_cont`
- :py:func:`datafusion.functions.approx_percentile_cont_with_weight`
+10. Grouping Set Functions
+ - :py:func:`datafusion.functions.grouping`
+ - :py:meth:`datafusion.expr.GroupingSet.rollup`
+ - :py:meth:`datafusion.expr.GroupingSet.cube`
+ - :py:meth:`datafusion.expr.GroupingSet.grouping_sets`
diff --git a/_sources/user-guide/common-operations/joins.rst.txt b/_sources/user-guide/common-operations/joins.rst.txt
index 1d9d7038..a289c937 100644
--- a/_sources/user-guide/common-operations/joins.rst.txt
+++ b/_sources/user-guide/common-operations/joins.rst.txt
@@ -134,3 +134,36 @@ In contrast to the above example, if we wish to get both columns:
.. ipython:: python
left.join(right, "id", how="inner", coalesce_duplicate_keys=False)
+
+Disambiguating Columns with ``DataFrame.col()``
+------------------------------------------------
+
+When both DataFrames contain non-key columns with the same name, you can use
+:py:meth:`~datafusion.dataframe.DataFrame.col` on each DataFrame **before** the
+join to create fully qualified column references. These references can then be
+used in the join predicate and when selecting from the result.
+
+This is especially useful with :py:meth:`~datafusion.dataframe.DataFrame.join_on`,
+which accepts expression-based predicates.
+
+.. ipython:: python
+
+ left = ctx.from_pydict(
+ {
+ "id": [1, 2, 3],
+ "val": [10, 20, 30],
+ }
+ )
+
+ right = ctx.from_pydict(
+ {
+ "id": [1, 2, 3],
+ "val": [40, 50, 60],
+ }
+ )
+
+ joined = left.join_on(
+ right, left.col("id") == right.col("id"), how="inner"
+ )
+
+ joined.select(left.col("id"), left.col("val"), right.col("val"))
diff --git a/_sources/user-guide/common-operations/windows.rst.txt b/_sources/user-guide/common-operations/windows.rst.txt
index c8fdea8f..d77881bc 100644
--- a/_sources/user-guide/common-operations/windows.rst.txt
+++ b/_sources/user-guide/common-operations/windows.rst.txt
@@ -175,10 +175,7 @@ it's ``Type 2`` column that are null.
Aggregate Functions
-------------------
-You can use any :ref:`Aggregation Function<aggregation>` as a window function. Currently
-aggregate functions must use the deprecated
-:py:func:`datafusion.functions.window` API but this should be resolved in
-DataFusion 42.0 (`Issue Link <https://github.com/apache/datafusion-python/issues/833>`_). Here
+You can use any :ref:`Aggregation Function<aggregation>` as a window function. Here
is an example that shows how to compare each pokemons’s attack power with the average attack
power in its ``"Type 1"`` using the :py:func:`datafusion.functions.avg` function.
@@ -189,10 +186,12 @@ power in its ``"Type 1"`` using the :py:func:`datafusion.functions.avg` function
col('"Name"'),
col('"Attack"'),
col('"Type 1"'),
- f.window("avg", [col('"Attack"')])
- .partition_by(col('"Type 1"'))
- .build()
- .alias("Average Attack"),
+ f.avg(col('"Attack"')).over(
+ Window(
+ window_frame=WindowFrame("rows", None, None),
+ partition_by=[col('"Type 1"')],
+ )
+ ).alias("Average Attack"),
)
Available Functions
diff --git a/_sources/user-guide/data-sources.rst.txt b/_sources/user-guide/data-sources.rst.txt
index 26f1303c..48ff4c01 100644
--- a/_sources/user-guide/data-sources.rst.txt
+++ b/_sources/user-guide/data-sources.rst.txt
@@ -224,25 +224,37 @@ A common technique for organizing tables is using a three level hierarchical app
supports this form of organizing using the :py:class:`~datafusion.catalog.Catalog`,
:py:class:`~datafusion.catalog.Schema`, and :py:class:`~datafusion.catalog.Table`. By default,
a :py:class:`~datafusion.context.SessionContext` comes with a single Catalog and a single Schema
-with the names ``datafusion`` and ``default``, respectively.
+with the names ``datafusion`` and ``public``, respectively.
The default implementation uses an in-memory approach to the catalog and schema. We have support
-for adding additional in-memory catalogs and schemas. This can be done like in the following
+for adding additional in-memory catalogs and schemas. You can access tables registered in a schema
+either through the Dataframe API or via sql commands. This can be done like in the following
example:
.. code-block:: python
+ import pyarrow as pa
from datafusion.catalog import Catalog, Schema
+ from datafusion import SessionContext
+
+ ctx = SessionContext()
my_catalog = Catalog.memory_catalog()
- my_schema = Schema.memory_schema()
+ my_schema = Schema.memory_schema()
+ my_catalog.register_schema('my_schema_name', my_schema)
+ ctx.register_catalog_provider('my_catalog_name', my_catalog)
- my_catalog.register_schema("my_schema_name", my_schema)
+ # Create an in-memory table
+ table = pa.table({
+ 'name': ['Bulbasaur', 'Charmander', 'Squirtle'],
+ 'type': ['Grass', 'Fire', 'Water'],
+ 'hp': [45, 39, 44],
+ })
+ df = ctx.create_dataframe([table.to_batches()], name='pokemon')
- ctx.register_catalog("my_catalog_name", my_catalog)
+ my_schema.register_table('pokemon', df)
-You could then register tables in ``my_schema`` and access them either through the DataFrame
-API or via sql commands such as ``"SELECT * from my_catalog_name.my_schema_name.my_table"``.
+ ctx.sql('SELECT * FROM my_catalog_name.my_schema_name.pokemon').show()
User Defined Catalog and Schema
-------------------------------
diff --git a/_sources/user-guide/dataframe/rendering.rst.txt b/_sources/user-guide/dataframe/rendering.rst.txt
index 9dea948b..dc61a422 100644
--- a/_sources/user-guide/dataframe/rendering.rst.txt
+++ b/_sources/user-guide/dataframe/rendering.rst.txt
@@ -15,18 +15,18 @@
.. specific language governing permissions and limitations
.. under the License.
-HTML Rendering in Jupyter
-=========================
+DataFrame Rendering
+===================
-When working in Jupyter notebooks or other environments that support rich HTML display,
-DataFusion DataFrames automatically render as nicely formatted HTML tables. This functionality
-is provided by the ``_repr_html_`` method, which is automatically called by Jupyter to provide
-a richer visualization than plain text output.
+DataFusion provides configurable rendering for DataFrames in both plain text and HTML
+formats. The ``datafusion.dataframe_formatter`` module controls how DataFrames are
+displayed in Jupyter notebooks (via ``_repr_html_``), in the terminal (via ``__repr__``),
+and anywhere else a string or HTML representation is needed.
-Basic HTML Rendering
---------------------
+Basic Rendering
+---------------
-In a Jupyter environment, simply displaying a DataFrame object will trigger HTML rendering:
+In a Jupyter environment, displaying a DataFrame triggers HTML rendering:
.. code-block:: python
@@ -36,74 +36,117 @@ In a Jupyter environment, simply displaying a DataFrame object will trigger HTML
# Explicit display also uses HTML rendering
display(df)
-Customizing HTML Rendering
----------------------------
+In a terminal or when converting to string, plain text rendering is used:
+
+.. code-block:: python
-DataFusion provides extensive customization options for HTML table rendering through the
-``datafusion.html_formatter`` module.
+ # Plain text table output
+ print(df)
-Configuring the HTML Formatter
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Configuring the Formatter
+-------------------------
-You can customize how DataFrames are rendered by configuring the formatter:
+You can customize how DataFrames are rendered by configuring the global formatter:
.. code-block:: python
- from datafusion.html_formatter import configure_formatter
-
- # Change the default styling
+ from datafusion.dataframe_formatter import configure_formatter
+
configure_formatter(
- max_cell_length=25, # Maximum characters in a cell before truncation
- max_width=1000, # Maximum width in pixels
- max_height=300, # Maximum height in pixels
- max_memory_bytes=2097152, # Maximum memory for rendering (2MB)
- min_rows=10, # Minimum number of rows to display
- max_rows=10, # Maximum rows to display in __repr__
- enable_cell_expansion=True,# Allow expanding truncated cells
- custom_css=None, # Additional custom CSS
+ max_cell_length=25, # Maximum characters in a cell before truncation
+ max_width=1000, # Maximum width in pixels (HTML only)
+ max_height=300, # Maximum height in pixels (HTML only)
+ max_memory_bytes=2097152, # Maximum memory for rendering (2MB)
+ min_rows=10, # Minimum number of rows to display
+ max_rows=10, # Maximum rows to display
+ enable_cell_expansion=True, # Allow expanding truncated cells (HTML only)
+ custom_css=None, # Additional custom CSS (HTML only)
show_truncation_message=True, # Show message when data is truncated
- style_provider=None, # Custom styling provider
- use_shared_styles=True # Share styles across tables
+ style_provider=None, # Custom styling provider (HTML only)
+ use_shared_styles=True, # Share styles across tables (HTML only)
)
The formatter settings affect all DataFrames displayed after configuration.
Custom Style Providers
------------------------
+----------------------
-For advanced styling needs, you can create a custom style provider:
+For HTML styling, you can create a custom style provider that implements the
+``StyleProvider`` protocol:
.. code-block:: python
- from datafusion.html_formatter import StyleProvider, configure_formatter
-
- class MyStyleProvider(StyleProvider):
- def get_table_styles(self):
- return {
- "table": "border-collapse: collapse; width: 100%;",
- "th": "background-color: #007bff; color: white; padding: 8px; text-align: left;",
- "td": "border: 1px solid #ddd; padding: 8px;",
- "tr:nth-child(even)": "background-color: #f2f2f2;",
- }
-
- def get_value_styles(self, dtype, value):
- """Return custom styles for specific values"""
- if dtype == "float" and value < 0:
- return "color: red;"
- return None
-
+ from datafusion.dataframe_formatter import configure_formatter
+
+ class MyStyleProvider:
+ def get_cell_style(self):
+ """Return CSS style string for table data cells."""
+ return "border: 1px solid #ddd; padding: 8px; text-align: left;"
+
+ def get_header_style(self):
+ """Return CSS style string for table header cells."""
+ return (
+ "background-color: #007bff; color: white; "
+ "padding: 8px; text-align: left;"
+ )
+
# Apply the custom style provider
configure_formatter(style_provider=MyStyleProvider())
+Custom Cell Formatters
+----------------------
+
+You can register custom formatters for specific Python types. A cell formatter is any
+callable that takes a value and returns a string:
+
+.. code-block:: python
+
+ from datafusion.dataframe_formatter import get_formatter
+
+ formatter = get_formatter()
+
+ # Format floats to 2 decimal places
+ formatter.register_formatter(float, lambda v: f"{v:.2f}")
+
+ # Format dates in a custom way
+ from datetime import date
+ formatter.register_formatter(date, lambda v: v.strftime("%B %d, %Y"))
+
+Custom Cell and Header Builders
+-------------------------------
+
+For full control over the HTML of individual cells or headers, you can set custom
+builder functions:
+
+.. code-block:: python
+
+ from datafusion.dataframe_formatter import get_formatter
+
+ formatter = get_formatter()
+
+ # Custom cell builder receives (value, row, col, table_id) and returns HTML
+ def my_cell_builder(value, row, col, table_id):
+ color = "red" if isinstance(value, (int, float)) and value < 0 else "black"
+ return f"<td style='color: {color}; padding: 8px;'>{value}</td>"
+
+ formatter.set_custom_cell_builder(my_cell_builder)
+
+ # Custom header builder receives a schema field and returns HTML
+ def my_header_builder(field):
+ return f"<th style='background: #333; color: white; padding: 8px;'>{field.name}</th>"
+
+ formatter.set_custom_header_builder(my_header_builder)
+
Performance Optimization with Shared Styles
--------------------------------------------
-The ``use_shared_styles`` parameter (enabled by default) optimizes performance when displaying
-multiple DataFrames in notebook environments:
+The ``use_shared_styles`` parameter (enabled by default) optimizes performance when
+displaying multiple DataFrames in notebook environments:
.. code-block:: python
- from datafusion.html_formatter import StyleProvider, configure_formatter
+ from datafusion.dataframe_formatter import configure_formatter
+
# Default: Use shared styles (recommended for notebooks)
configure_formatter(use_shared_styles=True)
@@ -111,76 +154,48 @@ multiple DataFrames in notebook environments:
configure_formatter(use_shared_styles=False)
When ``use_shared_styles=True``:
+
- CSS styles and JavaScript are included only once per notebook session
- This reduces HTML output size and prevents style duplication
- Improves rendering performance with many DataFrames
- Applies consistent styling across all DataFrames
-Creating a Custom Formatter
-----------------------------
+Working with the Formatter Directly
+------------------------------------
-For complete control over rendering, you can implement a custom formatter:
+You can use ``get_formatter()`` and ``set_formatter()`` for direct access to the global
+formatter instance:
.. code-block:: python
- from datafusion.html_formatter import Formatter, get_formatter
-
- class MyFormatter(Formatter):
- def format_html(self, batches, schema, has_more=False, table_uuid=None):
- # Create your custom HTML here
- html = "<div class='my-custom-table'>"
- # ... formatting logic ...
- html += "</div>"
- return html
-
- # Set as the global formatter
- configure_formatter(formatter_class=MyFormatter)
-
- # Or use the formatter just for specific operations
+ from datafusion.dataframe_formatter import (
+ DataFrameHtmlFormatter,
+ get_formatter,
+ set_formatter,
+ )
+
+ # Get and modify the current formatter
formatter = get_formatter()
- custom_html = formatter.format_html(batches, schema)
+ print(formatter.max_rows)
+ print(formatter.max_cell_length)
-Managing Formatters
--------------------
+ # Create and set a fully custom formatter
+ custom_formatter = DataFrameHtmlFormatter(
+ max_cell_length=50,
+ max_rows=20,
+ enable_cell_expansion=False,
+ )
+ set_formatter(custom_formatter)
Reset to default formatting:
.. code-block:: python
- from datafusion.html_formatter import reset_formatter
-
+ from datafusion.dataframe_formatter import reset_formatter
+
# Reset to default settings
reset_formatter()
-Get the current formatter settings:
-
-.. code-block:: python
-
- from datafusion.html_formatter import get_formatter
-
- formatter = get_formatter()
- print(formatter.max_rows)
- print(formatter.theme)
-
-Contextual Formatting
-----------------------
-
-You can also use a context manager to temporarily change formatting settings:
-
-.. code-block:: python
-
- from datafusion.html_formatter import formatting_context
-
- # Default formatting
- df.show()
-
- # Temporarily use different formatting
- with formatting_context(max_rows=100, theme="dark"):
- df.show() # Will use the temporary settings
-
- # Back to default formatting
- df.show()
-
Memory and Display Controls
---------------------------
@@ -188,10 +203,12 @@ You can control how much data is displayed and how much memory is used for rende
.. code-block:: python
+ from datafusion.dataframe_formatter import configure_formatter
+
configure_formatter(
max_memory_bytes=4 * 1024 * 1024, # 4MB maximum memory for display
min_rows=20, # Always show at least 20 rows
- max_rows=50 # Show up to 50 rows in output
+ max_rows=50, # Show up to 50 rows in output
)
These parameters help balance comprehensive data display against performance considerations.
@@ -216,7 +233,7 @@ Additional Resources
* :doc:`../io/index` - I/O Guide for reading data from various sources
* :doc:`../data-sources` - Comprehensive data sources guide
* :ref:`io_csv` - CSV file reading
-* :ref:`io_parquet` - Parquet file reading
+* :ref:`io_parquet` - Parquet file reading
* :ref:`io_json` - JSON file reading
* :ref:`io_avro` - Avro file reading
* :ref:`io_custom_table_provider` - Custom table providers
diff --git a/_sources/user-guide/upgrade-guides.rst.txt b/_sources/user-guide/upgrade-guides.rst.txt
index a77f6077..2ac7f770 100644
--- a/_sources/user-guide/upgrade-guides.rst.txt
+++ b/_sources/user-guide/upgrade-guides.rst.txt
@@ -18,6 +18,27 @@
Upgrade Guides
==============
+DataFusion 53.0.0
+-----------------
+
+This version includes an upgraded version of ``pyo3``, which changed the way to extract an FFI
+object. Example:
+
+Before:
+
+.. code-block:: rust
+
+ let codec = unsafe { capsule.reference::<FFI_LogicalExtensionCodec>() };
+
+Now:
+
+.. code-block:: rust
+
+ let data: NonNull<FFI_LogicalExtensionCodec> = capsule
+ .pointer_checked(Some(c_str!("datafusion_logical_extension_codec")))?
+ .cast();
+ let codec = unsafe { data.as_ref() };
+
DataFusion 52.0.0
-----------------
@@ -31,8 +52,10 @@ parameter, which is a Python object that can be used to extract the
``FFI_LogicalExtensionCodec`` that is necessary.
A complete example can be found in the `FFI example <https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-example>`_.
-Your methods need to be updated to take an additional parameter like in this
-example.
+Your FFI hook methods — ``__datafusion_catalog_provider__``,
+``__datafusion_schema_provider__``, ``__datafusion_table_provider__``, and
+``__datafusion_table_function__`` — need to be updated to accept an additional
+``session: Bound<PyAny>`` parameter, as shown in this example.
.. code-block:: rust
diff --git a/autoapi/datafusion/catalog/index.html b/autoapi/datafusion/catalog/index.html
index 2defa8e9..81f9290f 100644
--- a/autoapi/datafusion/catalog/index.html
+++ b/autoapi/datafusion/catalog/index.html
@@ -129,7 +129,7 @@
<ul>
<li class="toctree-l2">
<a class="reference internal" href="../../../user-guide/dataframe/rendering.html">
- HTML Rendering in Jupyter
+ DataFrame Rendering
</a>
</li>
</ul>
@@ -322,11 +322,6 @@
datafusion.functions
</a>
</li>
- <li class="toctree-l3">
- <a class="reference internal" href="../html_formatter/index.html">
- datafusion.html_formatter
- </a>
- </li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../input/index.html">
datafusion.input
@@ -445,15 +440,6 @@
</code>
</a>
</li>
- <li class="toc-h4 nav-item toc-entry">
- <a class="reference internal nav-link" href="#datafusion.catalog.Catalog.database">
- <code class="docutils literal notranslate">
- <span class="pre">
- Catalog.database()
- </span>
- </code>
- </a>
- </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.catalog.Catalog.deregister_schema">
<code class="docutils literal notranslate">
@@ -967,12 +953,6 @@
<dd><p>Print a string representation of the catalog.</p>
</dd></dl>
-<dl class="py method">
-<dt class="sig sig-object py" id="datafusion.catalog.Catalog.database">
-<span class="sig-name descname"><span class="pre">database</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'public'</span></span></em><span class="sig-par [...]
-<dd><p>Returns the database with the given <code class="docutils literal notranslate"><span class="pre">name</span></code> from this catalog.</p>
-</dd></dl>
-
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.catalog.Catalog.deregister_schema">
<span class="sig-name descname"><span class="pre">deregister_schema</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">cascade</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span cl [...]
diff --git a/autoapi/datafusion/context/index.html b/autoapi/datafusion/context/index.html
index fba3608b..3ffe7ecf 100644
--- a/autoapi/datafusion/context/index.html
+++ b/autoapi/datafusion/context/index.html
@@ -129,7 +129,7 @@
<ul>
<li class="toctree-l2">
<a class="reference internal" href="../../../user-guide/dataframe/rendering.html">
- HTML Rendering in Jupyter
+ DataFrame Rendering
</a>
</li>
</ul>
@@ -322,11 +322,6 @@
datafusion.functions
</a>
</li>
- <li class="toctree-l3">
- <a class="reference internal" href="../html_formatter/index.html">
- datafusion.html_formatter
- </a>
- </li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../input/index.html">
datafusion.input
@@ -467,15 +462,6 @@
</li>
</ul>
</li>
- <li class="toc-h3 nav-item toc-entry">
- <a class="reference internal nav-link" href="#datafusion.context.RuntimeConfig">
- <code class="docutils literal notranslate">
- <span class="pre">
- RuntimeConfig
- </span>
- </code>
- </a>
- </li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.context.RuntimeEnvBuilder">
<code class="docutils literal notranslate">
@@ -651,6 +637,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionConfig.with_extension">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionConfig.with_extension()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.context.SessionConfig.with_information_schema">
<code class="docutils literal notranslate">
@@ -833,6 +828,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.deregister_object_store">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.deregister_object_store()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.context.SessionContext.deregister_table">
<code class="docutils literal notranslate">
@@ -842,6 +846,42 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.deregister_udaf">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.deregister_udaf()
+ </span>
+ </code>
+ </a>
+ </li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.deregister_udf">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.deregister_udf()
+ </span>
+ </code>
+ </a>
+ </li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.deregister_udtf">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.deregister_udtf()
+ </span>
+ </code>
+ </a>
+ </li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.deregister_udwf">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.deregister_udwf()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.context.SessionContext.empty_table">
<code class="docutils literal notranslate">
@@ -851,6 +891,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.enable_ident_normalization">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.enable_ident_normalization()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.context.SessionContext.enable_url_table">
<code class="docutils literal notranslate">
@@ -870,19 +919,19 @@
</a>
</li>
<li class="toc-h4 nav-item toc-entry">
- <a class="reference internal nav-link" href="#datafusion.context.SessionContext.from_arrow">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.execute_logical_plan">
<code class="docutils literal notranslate">
<span class="pre">
- SessionContext.from_arrow()
+ SessionContext.execute_logical_plan()
</span>
</code>
</a>
</li>
<li class="toc-h4 nav-item toc-entry">
- <a class="reference internal nav-link" href="#datafusion.context.SessionContext.from_arrow_table">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.from_arrow">
<code class="docutils literal notranslate">
<span class="pre">
- SessionContext.from_arrow_table()
+ SessionContext.from_arrow()
</span>
</code>
</a>
@@ -932,6 +981,24 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.parse_sql_expr">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.parse_sql_expr()
+ </span>
+ </code>
+ </a>
+ </li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.read_arrow">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.read_arrow()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.context.SessionContext.read_avro">
<code class="docutils literal notranslate">
@@ -950,6 +1017,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.read_empty">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.read_empty()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.context.SessionContext.read_json">
<code class="docutils literal notranslate">
@@ -977,6 +1053,24 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.refresh_catalogs">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.refresh_catalogs()
+ </span>
+ </code>
+ </a>
+ </li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.register_arrow">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.register_arrow()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.context.SessionContext.register_avro">
<code class="docutils literal notranslate">
@@ -986,6 +1080,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.register_batch">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.register_batch()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.context.SessionContext.register_catalog_provider">
<code class="docutils literal notranslate">
@@ -1076,6 +1179,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.register_table_factory">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.register_table_factory()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.context.SessionContext.register_table_provider">
<code class="docutils literal notranslate">
@@ -1130,6 +1242,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.remove_optimizer_rule">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.remove_optimizer_rule()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.context.SessionContext.session_id">
<code class="docutils literal notranslate">
@@ -1139,6 +1260,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.session_start_time">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.session_start_time()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.context.SessionContext.sql">
<code class="docutils literal notranslate">
@@ -1175,6 +1305,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.context.SessionContext.table_provider">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.table_provider()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.context.SessionContext.with_logical_extension_codec">
<code class="docutils literal notranslate">
@@ -1251,22 +1390,19 @@
<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.context.ArrowStreamExportable" title="datafusion.context.ArrowStreamExportable"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ArrowStreamExportable</span></code></a></p></td>
<td><p>Type hint for object exporting Arrow C Stream via Arrow PyCapsule Interface.</p></td>
</tr>
-<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.context.RuntimeConfig" title="datafusion.context.RuntimeConfig"><code class="xref py py-obj docutils literal notranslate"><span class="pre">RuntimeConfig</span></code></a></p></td>
-<td><p>See <cite>RuntimeEnvBuilder</cite>.</p></td>
-</tr>
-<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.context.RuntimeEnvBuilder" title="datafusion.context.RuntimeEnvBuilder"><code class="xref py py-obj docutils literal notranslate"><span class="pre">RuntimeEnvBuilder</span></code></a></p></td>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.context.RuntimeEnvBuilder" title="datafusion.context.RuntimeEnvBuilder"><code class="xref py py-obj docutils literal notranslate"><span class="pre">RuntimeEnvBuilder</span></code></a></p></td>
<td><p>Runtime configuration options.</p></td>
</tr>
-<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.context.SQLOptions" title="datafusion.context.SQLOptions"><code class="xref py py-obj docutils literal notranslate"><span class="pre">SQLOptions</span></code></a></p></td>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.context.SQLOptions" title="datafusion.context.SQLOptions"><code class="xref py py-obj docutils literal notranslate"><span class="pre">SQLOptions</span></code></a></p></td>
<td><p>Options to be used when performing SQL queries.</p></td>
</tr>
-<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.context.SessionConfig" title="datafusion.context.SessionConfig"><code class="xref py py-obj docutils literal notranslate"><span class="pre">SessionConfig</span></code></a></p></td>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.context.SessionConfig" title="datafusion.context.SessionConfig"><code class="xref py py-obj docutils literal notranslate"><span class="pre">SessionConfig</span></code></a></p></td>
<td><p>Session configuration options.</p></td>
</tr>
-<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-obj docutils literal notranslate"><span class="pre">SessionContext</span></code></a></p></td>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-obj docutils literal notranslate"><span class="pre">SessionContext</span></code></a></p></td>
<td><p>This is the main interface for executing queries and creating DataFrames.</p></td>
</tr>
-<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.context.TableProviderExportable" title="datafusion.context.TableProviderExportable"><code class="xref py py-obj docutils literal notranslate"><span class="pre">TableProviderExportable</span></code></a></p></td>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.context.TableProviderExportable" title="datafusion.context.TableProviderExportable"><code class="xref py py-obj docutils literal notranslate"><span class="pre">TableProviderExportable</span></code></a></p></td>
<td><p>Type hint for object that has __datafusion_table_provider__ PyCapsule.</p></td>
</tr>
</tbody>
@@ -1300,14 +1436,6 @@
</dd></dl>
-<dl class="py class">
-<dt class="sig sig-object py" id="datafusion.context.RuntimeConfig">
-<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.context.</span></span><span class="sig-name descname"><span class="pre">RuntimeConfig</span></span><a class="headerlink" href="#datafusion.context.RuntimeConfig" title="Link to this definition">¶</a></dt>
-<dd><p>Bases: <a class="reference internal" href="#datafusion.context.RuntimeEnvBuilder" title="datafusion.context.RuntimeEnvBuilder"><code class="xref py py-obj docutils literal notranslate"><span class="pre">RuntimeEnvBuilder</span></code></a></p>
-<p>See <cite>RuntimeEnvBuilder</cite>.</p>
-<p>Create a new <a class="reference internal" href="#datafusion.context.RuntimeEnvBuilder" title="datafusion.context.RuntimeEnvBuilder"><code class="xref py py-class docutils literal notranslate"><span class="pre">RuntimeEnvBuilder</span></code></a> with default values.</p>
-</dd></dl>
-
<dl class="py class">
<dt class="sig sig-object py" id="datafusion.context.RuntimeEnvBuilder">
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.context.</span></span><span class="sig-name descname"><span class="pre">RuntimeEnvBuilder</span></span><a class="headerlink" href="#datafusion.context.RuntimeEnvBuilder" title="Link to this definition">¶</a></dt>
@@ -1375,8 +1503,8 @@ doing so:</p>
<dd class="field-even"><p>A new <a class="reference internal" href="#datafusion.context.RuntimeEnvBuilder" title="datafusion.context.RuntimeEnvBuilder"><code class="xref py py-class docutils literal notranslate"><span class="pre">RuntimeEnvBuilder</span></code></a> object with the updated setting.</p>
</dd>
</dl>
-<p>Examples usage:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">config</span> <span class="o">=</span> <span class="n">RuntimeEnvBuilder</span><span class="p">()</span><span class="o">.</span><span class="n">with_fair_spill_pool</span><span class="p">(</span><span class="mi">1024</span><span class="p">)</span>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">config</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">RuntimeEnvBuilder</span><span class="p">()</span><span class="o">.</span><span class="n">with_fair_spill_pool</span><span class="p">(</span><span class="mi">1024</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>
@@ -1396,8 +1524,8 @@ multiple spillable operators that all will spill.</p>
<dd class="field-even"><p>A new <a class="reference internal" href="#datafusion.context.RuntimeEnvBuilder" title="datafusion.context.RuntimeEnvBuilder"><code class="xref py py-class docutils literal notranslate"><span class="pre">RuntimeEnvBuilder</span></code></a> object with the updated setting.</p>
</dd>
</dl>
-<p>Example usage:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">config</span> <span class="o">=</span> <span class="n">RuntimeEnvBuilder</span><span class="p">()</span><span class="o">.</span><span class="n">with_greedy_memory_pool</span><span class="p">(</span><span class="mi">1024</span><span class="p">)</span>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">config</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">RuntimeEnvBuilder</span><span class="p">()</span><span class="o">.</span><span class="n">with_greedy_memory_pool</span><span class="p">(</span><span class="mi">1024</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>
@@ -1414,8 +1542,8 @@ multiple spillable operators that all will spill.</p>
<dd class="field-even"><p>A new <a class="reference internal" href="#datafusion.context.RuntimeEnvBuilder" title="datafusion.context.RuntimeEnvBuilder"><code class="xref py py-class docutils literal notranslate"><span class="pre">RuntimeEnvBuilder</span></code></a> object with the updated setting.</p>
</dd>
</dl>
-<p>Example usage:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">config</span> <span class="o">=</span> <span class="n">RuntimeEnvBuilder</span><span class="p">()</span><span class="o">.</span><span class="n">with_temp_file_path</span><span class="p">(</span><span class="s2">"/tmp"</span><span class="p">)</span>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">config</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">RuntimeEnvBuilder</span><span class="p">()</span><span class="o">.</span><span class="n">with_temp_file_path</span><span class="p">(</span><span class="s2">"/tmp"</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>
@@ -1460,8 +1588,8 @@ multiple spillable operators that all will spill.</p>
<dd class="field-even"><p>A new <a class="reference internal" href="#datafusion.context.SQLOptions" title="datafusion.context.SQLOptions"><code class="xref py py-class docutils literal notranslate"><span class="pre">SQLOptions</span></code></a> object with the updated setting.</p>
</dd>
</dl>
-<p>Example usage:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">options</span> <span class="o">=</span> <span class="n">SQLOptions</span><span class="p">()</span><span class="o">.</span><span class="n">with_allow_ddl</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">options</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SQLOptions</span><span class="p">()</span><span class="o">.</span><span class="n">with_allow_ddl</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>
@@ -1479,8 +1607,8 @@ multiple spillable operators that all will spill.</p>
<dd class="field-even"><p>A new <a class="reference internal" href="#datafusion.context.SQLOptions" title="datafusion.context.SQLOptions"><code class="xref py py-class docutils literal notranslate"><span class="pre">SQLOptions</span></code></a> object with the updated setting.</p>
</dd>
</dl>
-<p>Example usage:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">options</span> <span class="o">=</span> <span class="n">SQLOptions</span><span class="p">()</span><span class="o">.</span><span class="n">with_allow_dml</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">options</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SQLOptions</span><span class="p">()</span><span class="o">.</span><span class="n">with_allow_dml</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>
@@ -1500,8 +1628,8 @@ multiple spillable operators that all will spill.</p>
<dd class="field-odd"><p>A new</p>
</dd>
</dl>
-<p>Example usage:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">options</span> <span class="o">=</span> <span class="n">SQLOptions</span><span class="p">()</span><span class="o">.</span><span class="n">with_allow_statements</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">options</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SQLOptions</span><span class="p">()</span><span class="o">.</span><span class="n">with_allow_statements</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>
@@ -1583,6 +1711,23 @@ automatically created.</p>
</dl>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionConfig.with_extension">
+<span class="sig-name descname"><span class="pre">with_extension</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">extension</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href=" [...]
+<dd><p>Create a new configuration using an extension.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>extension</strong> – A custom configuration extension object. These are</p></li>
+<li><p><strong>library.</strong> (<em>shared from another DataFusion extension</em>)</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>A new <a class="reference internal" href="#datafusion.context.SessionConfig" title="datafusion.context.SessionConfig"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionConfig</span></code></a> object with the updated setting.</p>
+</dd>
+</dl>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionConfig.with_information_schema">
<span class="sig-name descname"><span class="pre">with_information_schema</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">enabled</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em><span [...]
@@ -1823,18 +1968,88 @@ a query against a CSV data source using the <code class="xref py py-class docuti
</dl>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.deregister_object_store">
+<span class="sig-name descname"><span class="pre">deregister_object_store</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">host</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><sp [...]
+<dd><p>Remove an object store from the session.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>schema</strong> – The data source schema (e.g. <code class="docutils literal notranslate"><span class="pre">"s3://"</span></code>).</p></li>
+<li><p><strong>host</strong> – URL for the host (e.g. bucket name).</p></li>
+</ul>
+</dd>
+</dl>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.deregister_table">
<span class="sig-name descname"><span class="pre">deregister_table</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></s [...]
<dd><p>Remove a table from the session.</p>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.deregister_udaf">
+<span class="sig-name descname"><span class="pre">deregister_udaf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></sp [...]
+<dd><p>Remove a user-defined aggregate function from the session.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>name</strong> – Name of the UDAF to deregister.</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.deregister_udf">
+<span class="sig-name descname"><span class="pre">deregister_udf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></spa [...]
+<dd><p>Remove a user-defined scalar function from the session.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>name</strong> – Name of the UDF to deregister.</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.deregister_udtf">
+<span class="sig-name descname"><span class="pre">deregister_udtf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></sp [...]
+<dd><p>Remove a user-defined table function from the session.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>name</strong> – Name of the UDTF to deregister.</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.deregister_udwf">
+<span class="sig-name descname"><span class="pre">deregister_udwf</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></sp [...]
+<dd><p>Remove a user-defined window function from the session.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>name</strong> – Name of the UDWF to deregister.</p>
+</dd>
+</dl>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.empty_table">
<span class="sig-name descname"><span class="pre">empty_table</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">datafusion.dataframe.DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.con [...]
<dd><p>Create an empty <a class="reference internal" href="../dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a>.</p>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.enable_ident_normalization">
+<span class="sig-name descname"><span class="pre">enable_ident_normalization</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">bool</span></span></span><a class="headerlink" href="#datafusion.context.SessionContext.enable_ident_normalization" title="Link to this definition">¶</a></dt>
+<dd><p>Return whether identifier normalization (lowercasing) is enabled.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">enable_ident_normalization</span><span class="p">()</span>
+<span class="go">True</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.enable_url_table">
<span class="sig-name descname"><span class="pre">enable_url_table</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><span class="pre">SessionContext</span></a></span></span><a class="headerlink" href="#datafusion.context.SessionContext.enable_u [...]
@@ -1852,6 +2067,34 @@ a query against a CSV data source using the <code class="xref py py-class docuti
<dd><p>Execute the <code class="docutils literal notranslate"><span class="pre">plan</span></code> and return the results.</p>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.execute_logical_plan">
+<span class="sig-name descname"><span class="pre">execute_logical_plan</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">plan</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../plan/index.html#datafusion.plan.LogicalPlan" title="datafusion.plan.LogicalPlan"><span class="pre">datafusion.plan.LogicalPlan</span></a></span></em><span class="sig-paren">) [...]
+<dd><p>Execute a <a class="reference internal" href="../plan/index.html#datafusion.plan.LogicalPlan" title="datafusion.plan.LogicalPlan"><code class="xref py py-class docutils literal notranslate"><span class="pre">LogicalPlan</span></code></a> and return a DataFrame.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>plan</strong> – Logical plan to execute.</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>DataFrame resulting from the execution.</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">plan</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">logical_plan</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df2</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">execute_logical_plan</span><span class="p">(</span><span class="n">plan</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">df2</span><span class="o">.</span><span class="n">collect</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+<span class="go"><pyarrow.lib.Int64Array object at ...></span>
+<span class="go">[</span>
+<span class="go"> 1,</span>
+<span class="go"> 2,</span>
+<span class="go"> 3</span>
+<span class="go">]</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.from_arrow">
<span class="sig-name descname"><span class="pre">from_arrow</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">data</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.context.ArrowStreamExportable" title="datafusion.context.ArrowStreamExportable"><span class="pre">ArrowStreamExportable</span></a><span class="w"> </span><span class="p"><span [...]
@@ -1873,13 +2116,6 @@ a struct array.</p>
</dl>
</dd></dl>
-<dl class="py method">
-<dt class="sig sig-object py" id="datafusion.context.SessionContext.from_arrow_table">
-<span class="sig-name descname"><span class="pre">from_arrow_table</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">data</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pyarrow.Table</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><s [...]
-<dd><p>Create a <a class="reference internal" href="../dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> from an Arrow table.</p>
-<p>This is an alias for <a class="reference internal" href="#datafusion.context.SessionContext.from_arrow" title="datafusion.context.SessionContext.from_arrow"><code class="xref py py-func docutils literal notranslate"><span class="pre">from_arrow()</span></code></a>.</p>
-</dd></dl>
-
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.from_pandas">
<span class="sig-name descname"><span class="pre">from_pandas</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">data</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">pandas.DataFrame</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><spa [...]
@@ -1959,6 +2195,98 @@ a struct array.</p>
</dl>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.parse_sql_expr">
+<span class="sig-name descname"><span class="pre">parse_sql_expr</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">sql</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class=" [...]
+<dd><p>Parse a SQL expression string into a logical expression.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>sql</strong> – SQL expression string.</p></li>
+<li><p><strong>schema</strong> – Schema to use for resolving column references.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>Parsed expression.</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion.common</span><span class="w"> </span><span class="kn">import</span> <span class="n">DFSchema</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">schema</span> <span class="o">=</span> <span class="n">DFSchema</span><span class="o">.</span><span class="n">empty</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">parse_sql_expr</span><span class="p">(</span><span class="s2">"1 + 2"</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="n">schema</span><span class="p">)</span>
+<span class="go">Expr(Int64(1) + Int64(2))</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.read_arrow">
+<span class="sig-name descname"><span class="pre">read_arrow</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span cla [...]
+<dd><p>Create a <code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code> for reading an Arrow IPC data source.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>path</strong> – Path to the Arrow IPC file.</p></li>
+<li><p><strong>schema</strong> – The data source schema.</p></li>
+<li><p><strong>file_extension</strong> – File extension to select.</p></li>
+<li><p><strong>file_partition_cols</strong> – Partition columns.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>DataFrame representation of the read Arrow IPC file.</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">tempfile</span><span class="o">,</span><span class="w"> </span><span class="nn">os</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">table</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">table</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmpdir</span><span class="p">:</span>
+<span class="gp">... </span> <span class="n">path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">tmpdir</span><span class="p">,</span> <span class="s2">"data.arrow"</span><span class="p">)</span>
+<span class="gp">... </span> <span class="k">with</span> <span class="n">pa</span><span class="o">.</span><span class="n">ipc</span><span class="o">.</span><span class="n">new_file</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span> <span class="k">as</span> <span class="n">writer</span><span class="p">:</span>
+<span class="gp">... </span> <span class="n">writer</span><span class="o">.</span><span class="n">write_table</span><span class="p">(</span><span class="n">table</span><span class="p">)</span>
+<span class="gp">... </span> <span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">read_arrow</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+<span class="gp">... </span> <span class="n">df</span><span class="o">.</span><span class="n">collect</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+<span class="go"><pyarrow.lib.Int64Array object at ...></span>
+<span class="go">[</span>
+<span class="go"> 1,</span>
+<span class="go"> 2,</span>
+<span class="go"> 3</span>
+<span class="go">]</span>
+</pre></div>
+</div>
+<p>Provide an explicit <code class="docutils literal notranslate"><span class="pre">schema</span></code> to override schema inference:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmpdir</span><span class="p">:</span>
+<span class="gp">... </span> <span class="n">path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">tmpdir</span><span class="p">,</span> <span class="s2">"data.arrow"</span><span class="p">)</span>
+<span class="gp">... </span> <span class="k">with</span> <span class="n">pa</span><span class="o">.</span><span class="n">ipc</span><span class="o">.</span><span class="n">new_file</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span> <span class="k">as</span> <span class="n">writer</span><span class="p">:</span>
+<span class="gp">... </span> <span class="n">writer</span><span class="o">.</span><span class="n">write_table</span><span class="p">(</span><span class="n">table</span><span class="p">)</span>
+<span class="gp">... </span> <span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">read_arrow</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">schema</span><span class="p">([(</span><span class="s2">"a"</span><span class="p">,</span> <span class="n">pa</span><span [...]
+<span class="gp">... </span> <span class="n">df</span><span class="o">.</span><span class="n">collect</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+<span class="go"><pyarrow.lib.Int64Array object at ...></span>
+<span class="go">[</span>
+<span class="go"> 1,</span>
+<span class="go"> 2,</span>
+<span class="go"> 3</span>
+<span class="go">]</span>
+</pre></div>
+</div>
+<p>Use <code class="docutils literal notranslate"><span class="pre">file_extension</span></code> to read files with a non-default extension:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmpdir</span><span class="p">:</span>
+<span class="gp">... </span> <span class="n">path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">tmpdir</span><span class="p">,</span> <span class="s2">"data.ipc"</span><span class="p">)</span>
+<span class="gp">... </span> <span class="k">with</span> <span class="n">pa</span><span class="o">.</span><span class="n">ipc</span><span class="o">.</span><span class="n">new_file</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span> <span class="k">as</span> <span class="n">writer</span><span class="p">:</span>
+<span class="gp">... </span> <span class="n">writer</span><span class="o">.</span><span class="n">write_table</span><span class="p">(</span><span class="n">table</span><span class="p">)</span>
+<span class="gp">... </span> <span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">read_arrow</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">file_extension</span><span class="o">=</span><span class="s2">".ipc"</span><span class="p">)</span>
+<span class="gp">... </span> <span class="n">df</span><span class="o">.</span><span class="n">collect</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+<span class="go"><pyarrow.lib.Int64Array object at ...></span>
+<span class="go">[</span>
+<span class="go"> 1,</span>
+<span class="go"> 2,</span>
+<span class="go"> 3</span>
+<span class="go">]</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.read_avro">
<span class="sig-name descname"><span class="pre">read_avro</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span clas [...]
@@ -2008,6 +2336,16 @@ combined with any of the other options in this method.</p></li>
</dl>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.read_empty">
+<span class="sig-name descname"><span class="pre">read_empty</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">datafusion.dataframe.DataFrame</span></a></span></span><a class="headerlink" href="#datafusion.cont [...]
+<dd><p>Create an empty <code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code> with no columns or rows.</p>
+<div class="admonition seealso">
+<p class="admonition-title">See also</p>
+<p>This is an alias for <a class="reference internal" href="#datafusion.context.SessionContext.empty_table" title="datafusion.context.SessionContext.empty_table"><code class="xref py py-meth docutils literal notranslate"><span class="pre">empty_table()</span></code></a>.</p>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.read_json">
<span class="sig-name descname"><span class="pre">read_json</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span clas [...]
@@ -2067,6 +2405,90 @@ specified as a column name (<code class="docutils literal notranslate"><span cla
<dd><p>Creates a <a class="reference internal" href="../dataframe/index.html#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> from a table.</p>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.refresh_catalogs">
+<span class="sig-name descname"><span class="pre">refresh_catalogs</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#datafusion.context.SessionContext.refresh_catalogs" title="Link to this definition">¶</a></dt>
+<dd><p>Refresh catalog metadata.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">refresh_catalogs</span><span class="p">()</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.register_arrow">
+<span class="sig-name descname"><span class="pre">register_arrow</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="p [...]
+<dd><p>Register an Arrow IPC file as a table.</p>
+<p>The registered table can be referenced from SQL statements executed
+against this context.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>name</strong> – Name of the table to register.</p></li>
+<li><p><strong>path</strong> – Path to the Arrow IPC file.</p></li>
+<li><p><strong>schema</strong> – The data source schema.</p></li>
+<li><p><strong>file_extension</strong> – File extension to select.</p></li>
+<li><p><strong>table_partition_cols</strong> – Partition columns.</p></li>
+</ul>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">tempfile</span><span class="o">,</span><span class="w"> </span><span class="nn">os</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">table</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">table</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmpdir</span><span class="p">:</span>
+<span class="gp">... </span> <span class="n">path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">tmpdir</span><span class="p">,</span> <span class="s2">"data.arrow"</span><span class="p">)</span>
+<span class="gp">... </span> <span class="k">with</span> <span class="n">pa</span><span class="o">.</span><span class="n">ipc</span><span class="o">.</span><span class="n">new_file</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span> <span class="k">as</span> <span class="n">writer</span><span class="p">:</span>
+<span class="gp">... </span> <span class="n">writer</span><span class="o">.</span><span class="n">write_table</span><span class="p">(</span><span class="n">table</span><span class="p">)</span>
+<span class="gp">... </span> <span class="n">ctx</span><span class="o">.</span><span class="n">register_arrow</span><span class="p">(</span><span class="s2">"arrow_tbl"</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span>
+<span class="gp">... </span> <span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT * FROM arrow_tbl"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+<span class="go"><pyarrow.lib.Int64Array object at ...></span>
+<span class="go">[</span>
+<span class="go"> 10,</span>
+<span class="go"> 20,</span>
+<span class="go"> 30</span>
+<span class="go">]</span>
+</pre></div>
+</div>
+<p>Provide an explicit <code class="docutils literal notranslate"><span class="pre">schema</span></code> to override schema inference:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmpdir</span><span class="p">:</span>
+<span class="gp">... </span> <span class="n">path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">tmpdir</span><span class="p">,</span> <span class="s2">"data.arrow"</span><span class="p">)</span>
+<span class="gp">... </span> <span class="k">with</span> <span class="n">pa</span><span class="o">.</span><span class="n">ipc</span><span class="o">.</span><span class="n">new_file</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span> <span class="k">as</span> <span class="n">writer</span><span class="p">:</span>
+<span class="gp">... </span> <span class="n">writer</span><span class="o">.</span><span class="n">write_table</span><span class="p">(</span><span class="n">table</span><span class="p">)</span>
+<span class="gp">... </span> <span class="n">ctx</span><span class="o">.</span><span class="n">register_arrow</span><span class="p">(</span>
+<span class="gp">... </span> <span class="s2">"arrow_schema"</span><span class="p">,</span>
+<span class="gp">... </span> <span class="n">path</span><span class="p">,</span>
+<span class="gp">... </span> <span class="n">schema</span><span class="o">=</span><span class="n">pa</span><span class="o">.</span><span class="n">schema</span><span class="p">([(</span><span class="s2">"x"</span><span class="p">,</span> <span class="n">pa</span><span class="o">.</span><span class="n">int64</span><span class="p">())]),</span>
+<span class="gp">... </span> <span class="p">)</span>
+<span class="gp">... </span> <span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT * FROM arrow_schema"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+<span class="go"><pyarrow.lib.Int64Array object at ...></span>
+<span class="go">[</span>
+<span class="go"> 10,</span>
+<span class="go"> 20,</span>
+<span class="go"> 30</span>
+<span class="go">]</span>
+</pre></div>
+</div>
+<p>Use <code class="docutils literal notranslate"><span class="pre">file_extension</span></code> to read files with a non-default extension:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmpdir</span><span class="p">:</span>
+<span class="gp">... </span> <span class="n">path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">tmpdir</span><span class="p">,</span> <span class="s2">"data.ipc"</span><span class="p">)</span>
+<span class="gp">... </span> <span class="k">with</span> <span class="n">pa</span><span class="o">.</span><span class="n">ipc</span><span class="o">.</span><span class="n">new_file</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">table</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span> <span class="k">as</span> <span class="n">writer</span><span class="p">:</span>
+<span class="gp">... </span> <span class="n">writer</span><span class="o">.</span><span class="n">write_table</span><span class="p">(</span><span class="n">table</span><span class="p">)</span>
+<span class="gp">... </span> <span class="n">ctx</span><span class="o">.</span><span class="n">register_arrow</span><span class="p">(</span>
+<span class="gp">... </span> <span class="s2">"arrow_ipc"</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">file_extension</span><span class="o">=</span><span class="s2">".ipc"</span>
+<span class="gp">... </span> <span class="p">)</span>
+<span class="gp">... </span> <span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT * FROM arrow_ipc"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+<span class="go"><pyarrow.lib.Int64Array object at ...></span>
+<span class="go">[</span>
+<span class="go"> 10,</span>
+<span class="go"> 20,</span>
+<span class="go"> 30</span>
+<span class="go">]</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.register_avro">
<span class="sig-name descname"><span class="pre">register_avro</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pr [...]
@@ -2086,6 +2508,33 @@ this context.</p>
</dl>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.register_batch">
+<span class="sig-name descname"><span class="pre">register_batch</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">batch</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class=" [...]
+<dd><p>Register a single <code class="xref py py-class docutils literal notranslate"><span class="pre">pa.RecordBatch</span></code> as a table.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>name</strong> – Name of the resultant table.</p></li>
+<li><p><strong>batch</strong> – Record batch to register as a table.</p></li>
+</ul>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">batch</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatch</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">register_batch</span><span class="p">(</span><span class="s2">"batch_tbl"</span><span class="p">,</span> <span class="n">batch</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT * FROM batch_tbl"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+<span class="go"><pyarrow.lib.Int64Array object at ...></span>
+<span class="go">[</span>
+<span class="go"> 1,</span>
+<span class="go"> 2,</span>
+<span class="go"> 3</span>
+<span class="go">]</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.register_catalog_provider">
<span class="sig-name descname"><span class="pre">register_catalog_provider</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">provider</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n" [...]
@@ -2261,6 +2710,22 @@ this context.</p>
</dl>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.register_table_factory">
+<span class="sig-name descname"><span class="pre">register_table_factory</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">format</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">factory</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n">< [...]
+<dd><p>Register a <code class="xref py py-class docutils literal notranslate"><span class="pre">TableProviderFactoryExportable</span></code>.</p>
+<p>The registered factory can be referenced from SQL DDL statements executed
+against this context.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>format</strong> – The value to be used in <cite>STORED AS ${format}</cite> clause.</p></li>
+<li><p><strong>factory</strong> – A PyCapsule that implements <code class="xref py py-class docutils literal notranslate"><span class="pre">TableProviderFactoryExportable</span></code></p></li>
+</ul>
+</dd>
+</dl>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.register_table_provider">
<span class="sig-name descname"><span class="pre">register_table_provider</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">provider</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n">< [...]
@@ -2306,12 +2771,44 @@ this context.</p>
</dl>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.remove_optimizer_rule">
+<span class="sig-name descname"><span class="pre">remove_optimizer_rule</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">bool</span></spa [...]
+<dd><p>Remove an optimizer rule by name.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>name</strong> – Name of the optimizer rule to remove.</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>True if a rule with the given name was found and removed.</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">remove_optimizer_rule</span><span class="p">(</span><span class="s2">"nonexistent_rule"</span><span class="p">)</span>
+<span class="go">False</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.session_id">
<span class="sig-name descname"><span class="pre">session_id</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.context.SessionContext.session_id" title="Link to this definition">¶</a></dt>
<dd><p>Return an id that uniquely identifies this <a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionContext</span></code></a>.</p>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.session_start_time">
+<span class="sig-name descname"><span class="pre">session_start_time</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="headerlink" href="#datafusion.context.SessionContext.session_start_time" title="Link to this definition">¶</a></dt>
+<dd><p>Return the session start time as an RFC 3339 formatted string.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">session_start_time</span><span class="p">()</span>
+<span class="go">'2026-01-01T12:34:56.123456789+00:00'</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.sql">
<span class="sig-name descname"><span class="pre">sql</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">query</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">options</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference i [...]
@@ -2373,6 +2870,33 @@ after parsing.</p></li>
<dd><p>Return whether a table with the given name exists.</p>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.table_provider">
+<span class="sig-name descname"><span class="pre">table_provider</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../ca [...]
+<dd><p>Return the <a class="reference internal" href="../catalog/index.html#datafusion.catalog.Table" title="datafusion.catalog.Table"><code class="xref py py-class docutils literal notranslate"><span class="pre">Table</span></code></a> for the given table name.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>name</strong> – Name of the table.</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>The table provider.</p>
+</dd>
+<dt class="field-odd">Raises<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>KeyError</strong> – If the table is not found.</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">batch</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">RecordBatch</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">register_record_batches</span><span class="p">(</span><span class="s2">"my_table"</span><span class="p">,</span> <span class="p">[[</span><span class="n">batch</span><span class="p">]])</span>
+<span class="gp">>>> </span><span class="n">tbl</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">table_provider</span><span class="p">(</span><span class="s2">"my_table"</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">tbl</span><span class="o">.</span><span class="n">schema</span>
+<span class="go">x: int64</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.with_logical_extension_codec">
<span class="sig-name descname"><span class="pre">with_logical_extension_codec</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">codec</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference intern [...]
diff --git a/autoapi/datafusion/dataframe/index.html b/autoapi/datafusion/dataframe/index.html
index 75b0c0c4..aafe9814 100644
--- a/autoapi/datafusion/dataframe/index.html
+++ b/autoapi/datafusion/dataframe/index.html
@@ -129,7 +129,7 @@
<ul>
<li class="toctree-l2">
<a class="reference internal" href="../../../user-guide/dataframe/rendering.html">
- HTML Rendering in Jupyter
+ DataFrame Rendering
</a>
</li>
</ul>
@@ -322,11 +322,6 @@
datafusion.functions
</a>
</li>
- <li class="toctree-l3">
- <a class="reference internal" href="../html_formatter/index.html">
- datafusion.html_formatter
- </a>
- </li>
<li class="toctree-l3 has-children">
<a class="reference internal" href="../input/index.html">
datafusion.input
@@ -609,6 +604,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.col">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ DataFrame.col()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.collect">
<code class="docutils literal notranslate">
@@ -636,6 +640,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.column">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ DataFrame.column()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.count">
<code class="docutils literal notranslate">
@@ -672,6 +685,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.distinct_on">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ DataFrame.distinct_on()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.drop">
<code class="docutils literal notranslate">
@@ -744,6 +766,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.find_qualified_columns">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ DataFrame.find_qualified_columns()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.head">
<code class="docutils literal notranslate">
@@ -862,37 +893,37 @@
</a>
</li>
<li class="toc-h4 nav-item toc-entry">
- <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.select_columns">
+ <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.select_exprs">
<code class="docutils literal notranslate">
<span class="pre">
- DataFrame.select_columns()
+ DataFrame.select_exprs()
</span>
</code>
</a>
</li>
<li class="toc-h4 nav-item toc-entry">
- <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.select_exprs">
+ <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.show">
<code class="docutils literal notranslate">
<span class="pre">
- DataFrame.select_exprs()
+ DataFrame.show()
</span>
</code>
</a>
</li>
<li class="toc-h4 nav-item toc-entry">
- <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.show">
+ <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.sort">
<code class="docutils literal notranslate">
<span class="pre">
- DataFrame.show()
+ DataFrame.sort()
</span>
</code>
</a>
</li>
<li class="toc-h4 nav-item toc-entry">
- <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.sort">
+ <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.sort_by">
<code class="docutils literal notranslate">
<span class="pre">
- DataFrame.sort()
+ DataFrame.sort_by()
</span>
</code>
</a>
@@ -969,6 +1000,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.union_by_name">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ DataFrame.union_by_name()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.union_distinct">
<code class="docutils literal notranslate">
@@ -987,6 +1027,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.window">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ DataFrame.window()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.dataframe.DataFrame.with_column">
<code class="docutils literal notranslate">
@@ -1090,6 +1139,53 @@
</li>
</ul>
</li>
+ <li class="toc-h3 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.dataframe.ExplainFormat">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ ExplainFormat
+ </span>
+ </code>
+ </a>
+ <ul class="nav section-nav flex-column">
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.dataframe.ExplainFormat.GRAPHVIZ">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ ExplainFormat.GRAPHVIZ
+ </span>
+ </code>
+ </a>
+ </li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.dataframe.ExplainFormat.INDENT">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ ExplainFormat.INDENT
+ </span>
+ </code>
+ </a>
+ </li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.dataframe.ExplainFormat.PGJSON">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ ExplainFormat.PGJSON
+ </span>
+ </code>
+ </a>
+ </li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#datafusion.dataframe.ExplainFormat.TREE">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ ExplainFormat.TREE
+ </span>
+ </code>
+ </a>
+ </li>
+ </ul>
+ </li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#datafusion.dataframe.InsertOp">
<code class="docutils literal notranslate">
@@ -1433,13 +1529,16 @@
<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.dataframe.DataFrameWriteOptions" title="datafusion.dataframe.DataFrameWriteOptions"><code class="xref py py-obj docutils literal notranslate"><span class="pre">DataFrameWriteOptions</span></code></a></p></td>
<td><p>Writer options for DataFrame.</p></td>
</tr>
-<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.dataframe.InsertOp" title="datafusion.dataframe.InsertOp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">InsertOp</span></code></a></p></td>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.dataframe.ExplainFormat" title="datafusion.dataframe.ExplainFormat"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ExplainFormat</span></code></a></p></td>
+<td><p>Output format for explain plans.</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.dataframe.InsertOp" title="datafusion.dataframe.InsertOp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">InsertOp</span></code></a></p></td>
<td><p>Insert operation mode.</p></td>
</tr>
-<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.dataframe.ParquetColumnOptions" title="datafusion.dataframe.ParquetColumnOptions"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ParquetColumnOptions</span></code></a></p></td>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.dataframe.ParquetColumnOptions" title="datafusion.dataframe.ParquetColumnOptions"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ParquetColumnOptions</span></code></a></p></td>
<td><p>Parquet options for individual columns.</p></td>
</tr>
-<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.dataframe.ParquetWriterOptions" title="datafusion.dataframe.ParquetWriterOptions"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ParquetWriterOptions</span></code></a></p></td>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.dataframe.ParquetWriterOptions" title="datafusion.dataframe.ParquetWriterOptions"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ParquetWriterOptions</span></code></a></p></td>
<td><p>Advanced parquet writer options.</p></td>
</tr>
</tbody>
@@ -1607,10 +1706,23 @@ Arrow and can be found at:
<dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.aggregate">
<span class="sig-name descname"><span class="pre">aggregate</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">group_by</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">collections.abc.Sequence</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre [...]
<dd><p>Aggregates the rows of the current DataFrame.</p>
+<p>By default each unique combination of the <code class="docutils literal notranslate"><span class="pre">group_by</span></code> columns
+produces one row. To get multiple levels of subtotals in a
+single pass, pass a
+<a class="reference internal" href="../expr/index.html#datafusion.expr.GroupingSet" title="datafusion.expr.GroupingSet"><code class="xref py py-class docutils literal notranslate"><span class="pre">GroupingSet</span></code></a> expression
+(created via
+<a class="reference internal" href="../expr/index.html#datafusion.expr.GroupingSet.rollup" title="datafusion.expr.GroupingSet.rollup"><code class="xref py py-meth docutils literal notranslate"><span class="pre">rollup()</span></code></a>,
+<a class="reference internal" href="../expr/index.html#datafusion.expr.GroupingSet.cube" title="datafusion.expr.GroupingSet.cube"><code class="xref py py-meth docutils literal notranslate"><span class="pre">cube()</span></code></a>, or
+<a class="reference internal" href="../expr/index.html#datafusion.expr.GroupingSet.grouping_sets" title="datafusion.expr.GroupingSet.grouping_sets"><code class="xref py py-meth docutils literal notranslate"><span class="pre">grouping_sets()</span></code></a>)
+as the <code class="docutils literal notranslate"><span class="pre">group_by</span></code> argument. See the
+<a class="reference internal" href="../../../user-guide/common-operations/aggregations.html#aggregation"><span class="std std-ref">Aggregation</span></a> user guide for detailed examples.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>group_by</strong> – Sequence of expressions or column names to group by.</p></li>
+<li><p><strong>group_by</strong> – Sequence of expressions or column names to group
+by. A <a class="reference internal" href="../expr/index.html#datafusion.expr.GroupingSet" title="datafusion.expr.GroupingSet"><code class="xref py py-class docutils literal notranslate"><span class="pre">GroupingSet</span></code></a>
+expression may be included to produce multiple grouping
+levels (rollup, cube, or explicit grouping sets).</p></li>
<li><p><strong>aggs</strong> – Sequence of expressions to aggregate.</p></li>
</ul>
</dd>
@@ -1645,6 +1757,16 @@ Arrow and can be found at:
</dl>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.col">
+<span class="sig-name descname"><span class="pre">col</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index.ht [...]
+<dd><p>Alias for <a class="reference internal" href="#datafusion.dataframe.DataFrame.column" title="datafusion.dataframe.DataFrame.column"><code class="xref py py-meth docutils literal notranslate"><span class="pre">column()</span></code></a>.</p>
+<div class="admonition seealso">
+<p class="admonition-title">See also</p>
+<p><a class="reference internal" href="#datafusion.dataframe.DataFrame.column" title="datafusion.dataframe.DataFrame.column"><code class="xref py py-meth docutils literal notranslate"><span class="pre">column()</span></code></a></p>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.collect">
<span class="sig-name descname"><span class="pre">collect</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">pyarrow.RecordBatch</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.collec [...]
@@ -1682,6 +1804,46 @@ partitioning.</p>
</dl>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.column">
+<span class="sig-name descname"><span class="pre">column</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../expr/index [...]
+<dd><p>Return a fully qualified column expression for <code class="docutils literal notranslate"><span class="pre">name</span></code>.</p>
+<p>Resolves an unqualified column name against this DataFrame’s schema
+and returns an <code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code> whose underlying column reference
+includes the table qualifier. This is especially useful after joins,
+where the same column name may appear in multiple relations.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>name</strong> – Unqualified column name to look up.</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>A fully qualified column expression.</p>
+</dd>
+<dt class="field-odd">Raises<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>Exception</strong> – If the column is not found or is ambiguous (exists in
+multiple relations).</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<p>Resolve a column from a simple DataFrame:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">3</sp [...]
+<span class="gp">>>> </span><span class="n">expr</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="n">expr</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span>
+<span class="go">{'a': [1, 2]}</span>
+</pre></div>
+</div>
+<p>Resolve qualified columns after a join:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">left</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"id"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class=" [...]
+<span class="gp">>>> </span><span class="n">right</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"id"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"y"</span><span class="p">:</span> <span class="p">[</span><span class="mi">3 [...]
+<span class="gp">>>> </span><span class="n">joined</span> <span class="o">=</span> <span class="n">left</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">right</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="s2">"id"</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s2">"inner"</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">expr</span> <span class="o">=</span> <span class="n">joined</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="s2">"y"</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">joined</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">"id"</span><span class="p">,</span> <span class="n">expr</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"id"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span>
+<span class="go">{'id': [1, 2], 'y': [30, 40]}</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.count">
<span class="sig-name descname"><span class="pre">count</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">int</span></span></span><a class="headerlink" href="#datafusion.dataframe.DataFrame.count" title="Link to this definition">¶</a></dt>
@@ -1728,42 +1890,102 @@ for non-numeric datatypes.</p>
</dl>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.distinct_on">
+<span class="sig-name descname"><span class="pre">distinct_on</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">on_expr</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><span class="pre">datafusion.expr.E [...]
+<dd><p>Deduplicate rows based on specific columns.</p>
+<p>Returns a new DataFrame with one row per unique combination of the
+<code class="docutils literal notranslate"><span class="pre">on_expr</span></code> columns, keeping the first row per group as determined by
+<code class="docutils literal notranslate"><span class="pre">sort_expr</span></code>.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>on_expr</strong> – Expressions that determine uniqueness.</p></li>
+<li><p><strong>select_expr</strong> – Expressions to include in the output.</p></li>
+<li><p><strong>sort_expr</strong> – Optional sort expressions to determine which row to keep.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>DataFrame after deduplication.</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<p>Keep the row with the smallest <code class="docutils literal notranslate"><span class="pre">b</span></code> for each unique <code class="docutils literal notranslate"><span class="pre">a</span></code>:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">col</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span cl [...]
+<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">distinct_on</span><span class="p">(</span>
+<span class="gp">... </span> <span class="p">[</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)],</span>
+<span class="gp">... </span> <span class="p">[</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">),</span> <span class="n">col</span><span class="p">(</span><span class="s2">"b"</span><span class="p">)],</span>
+<span class="gp">... </span> <span class="p">[</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">ascending</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span> <span class="n">col</span><span class="p">(</span><span class="s2">"b"</span><span class="p">)</span><span class="o">.</span><span [...]
+<span class="gp">... </span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span>
+<span class="go">{'a': [1, 2], 'b': [10, 30]}</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.drop">
<span class="sig-name descname"><span class="pre">drop</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">columns</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint" [...]
<dd><p>Drop arbitrary amount of columns.</p>
-<p>Column names are case-sensitive and do not require double quotes like
-other operations such as <cite>select</cite>. Leading and trailing double quotes
-are allowed and will be automatically stripped if present.</p>
+<p>Column names are case-sensitive and require double quotes to be dropped
+if the original name is not strictly lower case.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
-<dd class="field-odd"><p><strong>columns</strong> – Column names to drop from the dataframe. Both <code class="docutils literal notranslate"><span class="pre">column_name</span></code>
-and <code class="docutils literal notranslate"><span class="pre">"column_name"</span></code> are accepted.</p>
+<dd class="field-odd"><p><strong>columns</strong> – Column names to drop from the dataframe.</p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>DataFrame with those columns removed in the projection.</p>
</dd>
</dl>
-<p>Example Usage:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s1">'ID_For_Students'</span><span class="p">)</span> <span class="c1"># Works</span>
-<span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s1">'"ID_For_Students"'</span><span class="p">)</span> <span class="c1"># Also works (quotes stripped)</span>
+<p class="rubric">Examples</p>
+<p>To drop a lower-cased column ‘a’</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">3</sp [...]
+<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span><span class="o">.</span><span class="n">names</span>
+<span class="go">['b']</span>
+</pre></div>
+</div>
+<p>Or to drop an upper-cased column ‘A’</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"A"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">3</sp [...]
+<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s1">'"A"'</span><span class="p">)</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span><span class="o">.</span><span class="n">names</span>
+<span class="go">['b']</span>
</pre></div>
</div>
</dd></dl>
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.except_all">
-<span class="sig-name descname"><span class="pre">except_all</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">other</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span [...]
-<dd><p>Calculate the exception of two <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a>.</p>
+<span class="sig-name descname"><span class="pre">except_all</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">other</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">distin [...]
+<dd><p>Calculate the set difference of two <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a>.</p>
+<p>Returns rows that are in this DataFrame but not in <code class="docutils literal notranslate"><span class="pre">other</span></code>.</p>
<p>The two <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> must have exactly the same schema.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
-<dd class="field-odd"><p><strong>other</strong> – DataFrame to calculate exception with.</p>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>other</strong> – DataFrame to calculate exception with.</p></li>
+<li><p><strong>distinct</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, duplicate rows are removed from the result.</p></li>
+</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
-<dd class="field-even"><p>DataFrame after exception.</p>
+<dd class="field-even"><p>DataFrame after set difference.</p>
</dd>
</dl>
+<p class="rubric">Examples</p>
+<p>Remove rows present in <code class="docutils literal notranslate"><span class="pre">df2</span></code>:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df1</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</s [...]
+<span class="gp">>>> </span><span class="n">df2</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</ [...]
+<span class="gp">>>> </span><span class="n">df1</span><span class="o">.</span><span class="n">except_all</span><span class="p">(</span><span class="n">df2</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span>
+<span class="go">{'a': [3], 'b': [30]}</span>
+</pre></div>
+</div>
+<p>Remove rows present in <code class="docutils literal notranslate"><span class="pre">df2</span></code> and deduplicate:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df1</span><span class="o">.</span><span class="n">except_all</span><span class="p">(</span><span class="n">df2</span><span class="p">,</span> <span class="n">distinct</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="s2">" [...]
+<span class="go">{'a': [3], 'b': [30]}</span>
+</pre></div>
+</div>
</dd></dl>
<dl class="py method">
@@ -1801,7 +2023,7 @@ and <code class="docutils literal notranslate"><span class="pre">"column_na
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.explain">
-<span class="sig-name descname"><span class="pre">explain</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">verbose</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-par [...]
+<span class="sig-name descname"><span class="pre">explain</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">verbose</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-par [...]
<dd><p>Print an explanation of the DataFrame’s plan so far.</p>
<p>If <code class="docutils literal notranslate"><span class="pre">analyze</span></code> is specified, runs the plan and reports metrics.</p>
<dl class="field-list simple">
@@ -1809,9 +2031,23 @@ and <code class="docutils literal notranslate"><span class="pre">"column_na
<dd class="field-odd"><ul class="simple">
<li><p><strong>verbose</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, more details will be included.</p></li>
<li><p><strong>analyze</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, the plan will run and metrics reported.</p></li>
+<li><p><strong>format</strong> – Output format for the plan. Defaults to
+<a class="reference internal" href="#datafusion.dataframe.ExplainFormat.INDENT" title="datafusion.dataframe.ExplainFormat.INDENT"><code class="xref py py-attr docutils literal notranslate"><span class="pre">ExplainFormat.INDENT</span></code></a>.</p></li>
</ul>
</dd>
</dl>
+<p class="rubric">Examples</p>
+<p>Show the plan in tree format:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">ExplainFormat</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">explain</span><span class="p">(</span><span class="nb">format</span><span class="o">=</span><span class="n">ExplainFormat</span><span class="o">.</span><span class="n">TREE</span><span class="p">)</span>
+</pre></div>
+</div>
+<p>Show plan with runtime metrics:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">explain</span><span class="p">(</span><span class="n">analyze</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+</pre></div>
+</div>
</dd></dl>
<dl class="py method">
@@ -1830,9 +2066,12 @@ and <code class="docutils literal notranslate"><span class="pre">"column_na
</dd>
</dl>
<p class="rubric">Examples</p>
-<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">fill_null</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> <span class="c1"># Fill all nulls with 0 where possible</span>
-<span class="gp">>>> </span><span class="c1"># Fill nulls in specific string columns</span>
-<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">fill_null</span><span class="p">(</span><span class="s2">"missing"</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="p">[</span><span class="s2">"name"</span><span class="p">,</span> <span class="s2">"category"</span><span class="p">])</span>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span><span class="p">,</span> <span class="n">col</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:< [...]
+<span class="gp">>>> </span><span class="n">filled</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">fill_null</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">filled</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">))</span><span class="o">.</span><span class="n">collect</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="s2">"a"</s [...]
+<span class="go">[0, 1, 3]</span>
</pre></div>
</div>
<p class="rubric">Notes</p>
@@ -1854,10 +2093,13 @@ combined as a logical AND. Each <code class="docutils literal notranslate"><span
<a class="reference internal" href="../index.html#datafusion.col" title="datafusion.col"><code class="xref py py-func docutils literal notranslate"><span class="pre">datafusion.col()</span></code></a> or <a class="reference internal" href="../index.html#datafusion.lit" title="datafusion.lit"><code class="xref py py-func docutils literal notranslate"><span class="pre">datafusion.lit()</span></code></a>, or a SQL expression string
that will be parsed against the DataFrame schema. If more complex logic is
required, see the logical operations in <a class="reference internal" href="../functions/index.html#module-datafusion.functions" title="datafusion.functions"><code class="xref py py-mod docutils literal notranslate"><span class="pre">functions</span></code></a>.</p>
-<p>Example:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">col</span><span class="p">,</span> <span class="n">lit</span>
-<span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span> <span class="o">></span> <span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
-<span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="s2">"a > 1"</span><span class="p">)</span>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span> <span class="o">></span> <span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span>
+<span class="go">{'a': [2, 3]}</span>
+<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="s2">"a > 1"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span>
+<span class="go">{'a': [2, 3]}</span>
</pre></div>
</div>
<dl class="field-list simple">
@@ -1870,6 +2112,35 @@ required, see the logical operations in <a class="reference internal" href="../f
</dl>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.find_qualified_columns">
+<span class="sig-name descname"><span class="pre">find_qualified_columns</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">names</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig- [...]
+<dd><p>Return fully qualified column expressions for the given names.</p>
+<p>This is a batch version of <a class="reference internal" href="#datafusion.dataframe.DataFrame.column" title="datafusion.dataframe.DataFrame.column"><code class="xref py py-meth docutils literal notranslate"><span class="pre">column()</span></code></a> — it resolves each
+unqualified name against the DataFrame’s schema and returns a list
+of qualified column expressions.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>names</strong> – Unqualified column names to look up.</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>List of fully qualified column expressions, one per name.</p>
+</dd>
+<dt class="field-odd">Raises<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>Exception</strong> – If any column is not found or is ambiguous.</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<p>Resolve multiple columns at once:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">3</sp [...]
+<span class="gp">>>> </span><span class="n">exprs</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">find_qualified_columns</span><span class="p">(</span><span class="s2">"a"</span><span class="p">,</span> <span class="s2">"c"</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="o">*</span><span class="n">exprs</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span>
+<span class="go">{'a': [1, 2], 'c': [5, 6]}</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.head">
<span class="sig-name descname"><span class="pre">head</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">n</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">5</span></span></em><span class="sig-paren">)</span> < [...]
@@ -1886,17 +2157,36 @@ required, see the logical operations in <a class="reference internal" href="../f
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.dataframe.DataFrame.intersect">
-<span class="sig-name descname"><span class="pre">intersect</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">other</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span c [...]
+<span class="sig-name descname"><span class="pre">intersect</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">other</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">distinc [...]
<dd><p>Calculate the intersection of two <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a>.</p>
<p>The two <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> must have exactly the same schema.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
-<dd class="field-odd"><p><strong>other</strong> – DataFrame to intersect with.</p>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>other</strong> – DataFrame to intersect with.</p></li>
+<li><p><strong>distinct</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, duplicate rows are removed from the result.</p></li>
+</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>DataFrame after intersection.</p>
</dd>
</dl>
+<p class="rubric">Examples</p>
+<p>Find rows common to both DataFrames:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df1</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</s [...]
+<span class="gp">>>> </span><span class="n">df2</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">4</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</ [...]
+<span class="gp">>>> </span><span class="n">df1</span><span class="o">.</span><span class="n">intersect</span><span class="p">(</span><span class="n">df2</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span>
+<span class="go">{'a': [1], 'b': [10]}</span>
+</pre></div>
+</div>
+<p>Intersect with deduplication:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">df1</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi" [...]
+<span class="gp">>>> </span><span class="n">df2</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">10</ [...]
+<span class="gp">>>> </span><span class="n">df1</span><span class="o">.</span><span class="n">intersect</span><span class="p">(</span><span class="n">df2</span><span class="p">,</span> <span class="n">distinct</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span>
+<span class="go">{'a': [1], 'b': [10]}</span>
+</pre></div>
+</div>
</dd></dl>
<dl class="py method">
@@ -1909,8 +2199,9 @@ required, see the logical operations in <a class="reference internal" href="../f
<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT 1 AS value"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">view</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">into_view</span><span class="p">()</span>
<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">register_table</span><span class="p">(</span><span class="s2">"values_view"</span><span class="p">,</span> <span class="n">view</span><span class="p">)</span>
-<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span> <span class="c1"># The DataFrame is still usable</span>
-<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT value FROM values_view"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">result</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT value FROM values_view"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">result</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="s2">"value"</span><span class="p">)</span><span class="o">.</span><span class="n">to_pylist</span><span class="p">()</span>
+<span class="go">[1]</span>
</pre></div>
</div>
</dd></dl>
@@ -1923,7 +2214,12 @@ required, see the logical operations in <a class="reference internal" href="../f
<dt class="sig sig-object py">
<span class="sig-name descname"><span class="pre">join</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">right</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><span class="pre">DataFrame</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">on</span></s [...]
<dd><p>Join this <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a> with another <a class="reference internal" href="#datafusion.dataframe.DataFrame" title="datafusion.dataframe.DataFrame"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code></a>.</p>
-<p><cite>on</cite> has to be provided or both <cite>left_on</cite> and <cite>right_on</cite> in conjunction.</p>
+<p><code class="docutils literal notranslate"><span class="pre">on</span></code> has to be provided or both <code class="docutils literal notranslate"><span class="pre">left_on</span></code> and <code class="docutils literal notranslate"><span class="pre">right_on</span></code> in
+conjunction.</p>
+<p>When non-key columns share the same name in both DataFrames, use
+<a class="reference internal" href="#datafusion.dataframe.DataFrame.col" title="datafusion.dataframe.DataFrame.col"><code class="xref py py-meth docutils literal notranslate"><span class="pre">DataFrame.col()</span></code></a> on each DataFrame <strong>before</strong> the join to
+obtain fully qualified column references that can disambiguate them.
+See <a class="reference internal" href="#datafusion.dataframe.DataFrame.join_on" title="datafusion.dataframe.DataFrame.join_on"><code class="xref py py-meth docutils literal notranslate"><span class="pre">join_on()</span></code></a> for an example.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
@@ -1952,9 +2248,31 @@ that have identical names in the <code class="docutils literal notranslate"><spa
<p>Join predicates must be <a class="reference internal" href="../expr/index.html#datafusion.expr.Expr" title="datafusion.expr.Expr"><code class="xref py py-class docutils literal notranslate"><span class="pre">Expr</span></code></a> objects, typically
built with <a class="reference internal" href="../index.html#datafusion.col" title="datafusion.col"><code class="xref py py-func docutils literal notranslate"><span class="pre">datafusion.col()</span></code></a>. On expressions are used to support
in-equality predicates. Equality predicates are correctly optimized.</p>
-<p>Example:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">col</span>
-<span class="n">df</span><span class="o">.</span><span class="n">join_on</span><span class="p">(</span><span class="n">other_df</span><span class="p">,</span> <span class="n">col</span><span class="p">(</span><span class="s2">"id"</span><span class="p">)</span> <span class="o">==</span> <span class="n">col</span><span class="p">(</span><span class="s2">"other_id"</span><span class="p">))</span>
+<p>Use <a class="reference internal" href="#datafusion.dataframe.DataFrame.col" title="datafusion.dataframe.DataFrame.col"><code class="xref py py-meth docutils literal notranslate"><span class="pre">DataFrame.col()</span></code></a> on each DataFrame <strong>before</strong> the join to
+obtain fully qualified column references. These qualified references
+can then be used in the join predicate and to disambiguate columns
+with the same name when selecting from the result.</p>
+<p class="rubric">Examples</p>
+<p>Join with unique column names:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">left</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"x"</span><span class="p">:</span> <span class="p">[</span><span class="s2">&qu [...]
+<span class="gp">>>> </span><span class="n">right</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"b"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"y"</span><span class="p">:</span> <span class="p">[</span><span class="s2">&q [...]
+<span class="gp">>>> </span><span class="n">left</span><span class="o">.</span><span class="n">join_on</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">right</span><span class="p">,</span> <span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span> <span class="o">==</span> <span class="n">col</span><span class="p">(</span><span class="s2">"b"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">col</span><span class="p">(</span><span class="s2">"x"</span><span class="p">))</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span>
+<span class="go">{'a': [1, 2], 'x': ['a', 'b'], 'b': [1, 2], 'y': ['c', 'd']}</span>
+</pre></div>
+</div>
+<p>Use <a class="reference internal" href="#datafusion.dataframe.DataFrame.col" title="datafusion.dataframe.DataFrame.col"><code class="xref py py-meth docutils literal notranslate"><span class="pre">col()</span></code></a> to disambiguate shared column names:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">left</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"id"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class=" [...]
+<span class="gp">>>> </span><span class="n">right</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"id"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="s2">"val"</span><span class="p">:</span> <span class="p">[</span><span class="mi" [...]
+<span class="gp">>>> </span><span class="n">joined</span> <span class="o">=</span> <span class="n">left</span><span class="o">.</span><span class="n">join_on</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">right</span><span class="p">,</span> <span class="n">left</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"id"</span><span class="p">)</span> <span class="o">==</span> <span class="n">right</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"id"</span><span class="p">),</span> <span class="n">how</span><span class="o">=</span>< [...]
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">joined</span><span class="o">.</span><span class="n">select</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">left</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"id"</span><span class="p">),</span> <span class="n">left</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"val"</span><span class="p">),</span> <span class="n">right</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"val& [...]
+<span class="gp">... </span><span class="p">)</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">left</span><span class="o">.</span><span class="n">col</span><span class="p">(</span><span class="s2">"id"</span><span class="p">))</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span>
+<span class="go">{'id': [1, 2], 'val': [10, 20], 'rval': [30, 40]}</span>
</pre></div>
</div>
<dl class="field-list simple">
@@ -2016,13 +2334,12 @@ in-equality predicates. Equality predicates are correctly optimized.</p>
<span class="sig-name descname"><span class="pre">parse_sql_expr</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">expr</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference internal" href="../ex [...]
<dd><p>Creates logical expression from a SQL query text.</p>
<p>The expression is created and processed against the current schema.</p>
-<p>Example:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">col</span><span class="p">,</span> <span class="n">lit</span>
-<span class="n">df</span><span class="o">.</span><span class="n">parse_sql_expr</span><span class="p">(</span><span class="s2">"a > 1"</span><span class="p">)</span>
-
-<span class="n">should</span> <span class="n">produce</span><span class="p">:</span>
-
-<span class="n">col</span><span class="p">(</span><span class="s2">"a"</span><span class="p">)</span> <span class="o">></span> <span class="n">lit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">dfn</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]})</span>
+<span class="gp">>>> </span><span class="n">expr</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">parse_sql_expr</span><span class="p">(</span><span class="s2">"a > 1"</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">df</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">expr</span><span class="p">)</span><span class="o">.</span><span class="n">to_pydict</span><span class="p">()</span>
+<span class="go">{'a': [2, 3]}</span>
</pre></div>
</div>
<dl class="field-list simple">
@@ -2103,17 +2420,6 @@ column <code class="docutils literal notranslate"><span class="pre">a</span></co
... 13215 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org