You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/08 20:05:17 UTC
(datafusion-python) branch asf-site updated: Publish built docs triggered by a2882b9fcfe2a366ded6c94f6d50057b69bd8889
This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/datafusion-python.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 2fd0e2ac Publish built docs triggered by a2882b9fcfe2a366ded6c94f6d50057b69bd8889
2fd0e2ac is described below
commit 2fd0e2ac6e256368ddca0883a3d887e5dd6a3698
Author: github-actions[bot] <gi...@users.noreply.github.com>
AuthorDate: Thu Oct 8 20:05:11 2026 +0000
Publish built docs triggered by a2882b9fcfe2a366ded6c94f6d50057b69bd8889
---
_sources/autoapi/datafusion/catalog/index.rst.txt | 17 +
_sources/autoapi/datafusion/context/index.rst.txt | 643 +++++++++++-
.../autoapi/datafusion/dataframe/index.rst.txt | 120 ++-
_sources/autoapi/datafusion/expr/index.rst.txt | 64 +-
.../autoapi/datafusion/extensions/index.rst.txt | 345 +++++++
.../autoapi/datafusion/functions/index.rst.txt | 615 ++++++++++--
.../datafusion/functions/spark/index.rst.txt | 242 ++++-
_sources/autoapi/datafusion/index.rst.txt | 368 ++++++-
_sources/autoapi/datafusion/ipc/index.rst.txt | 7 +-
_sources/autoapi/datafusion/plan/index.rst.txt | 249 ++++-
.../autoapi/datafusion/user_defined/index.rst.txt | 96 +-
_sources/contributor-guide/ffi-internals.md.txt | 185 ++++
_sources/contributor-guide/ffi.md.txt | 259 -----
_sources/contributor-guide/index.md.txt | 10 +-
_sources/contributor-guide/introduction.md.txt | 4 +-
_sources/contributor-guide/pyo3-guidelines.md.txt | 74 ++
_sources/extension-guide/bundles.md.txt | 313 ++++++
_sources/extension-guide/capsule-protocol.md.txt | 238 +++++
_sources/extension-guide/checklist.md.txt | 99 ++
_sources/extension-guide/codecs.md.txt | 183 ++++
_sources/extension-guide/functions.md.txt | 106 ++
_sources/extension-guide/index.md.txt | 136 +++
_sources/extension-guide/other-components.md.txt | 72 ++
_sources/extension-guide/query-planners.md.txt | 104 ++
_sources/extension-guide/sessions.md.txt | 117 +++
_sources/extension-guide/table-providers.md.txt | 143 +++
_sources/extension-guide/why-ffi.md.txt | 148 +++
_sources/index.md.txt | 11 +
.../common-operations/aggregations.md.txt | 3 +-
.../common-operations/expressions.md.txt | 5 +-
.../user-guide/common-operations/functions.md.txt | 35 +
.../common-operations/udf-and-udfa.md.txt | 3 +-
.../user-guide/common-operations/windows.md.txt | 71 +-
_sources/user-guide/configuration.md.txt | 84 ++
_sources/user-guide/data-sources.md.txt | 32 +-
.../expressions.md.txt} | 65 +-
_sources/user-guide/distributing-work/index.md.txt | 81 ++
.../distributing-work/query-engines.md.txt | 100 ++
_sources/user-guide/extensions.md.txt | 142 +++
_sources/user-guide/index.md.txt | 9 +-
_sources/user-guide/io/table_provider.md.txt | 20 +-
_sources/user-guide/upgrade-guides.md.txt | 417 ++++++++
_static/check-solid.svg | 4 +
_static/clipboard.min.js | 7 +
_static/copy-button.svg | 5 +
_static/copybutton.css | 94 ++
_static/copybutton.js | 248 +++++
_static/copybutton_funcs.js | 73 ++
autoapi/datafusion/catalog/index.html | 61 +-
autoapi/datafusion/context/index.html | 746 ++++++++++++--
autoapi/datafusion/dataframe/index.html | 181 +++-
autoapi/datafusion/dataframe_formatter/index.html | 41 +-
autoapi/datafusion/expr/index.html | 107 +-
autoapi/datafusion/extensions/index.html | 975 ++++++++++++++++++
autoapi/datafusion/functions/index.html | 1034 +++++++++++++++-----
autoapi/datafusion/functions/spark/index.html | 453 ++++++++-
autoapi/datafusion/index.html | 438 +++++++--
autoapi/datafusion/input/base/index.html | 41 +-
autoapi/datafusion/input/index.html | 41 +-
autoapi/datafusion/input/location/index.html | 41 +-
autoapi/datafusion/io/index.html | 41 +-
autoapi/datafusion/ipc/index.html | 48 +-
autoapi/datafusion/object_store/index.html | 41 +-
autoapi/datafusion/options/index.html | 41 +-
autoapi/datafusion/plan/index.html | 299 +++++-
autoapi/datafusion/record_batch/index.html | 41 +-
autoapi/datafusion/substrait/index.html | 41 +-
autoapi/datafusion/unparser/index.html | 41 +-
autoapi/datafusion/user_defined/index.html | 141 ++-
autoapi/index.html | 48 +-
.../{introduction.html => ffi-internals.html} | 310 +++---
contributor-guide/ffi.html | 803 +--------------
contributor-guide/index.html | 70 +-
contributor-guide/introduction.html | 53 +-
.../{index.html => pyo3-guidelines.html} | 159 ++-
extension-guide/bundles.html | 894 +++++++++++++++++
extension-guide/capsule-protocol.html | 826 ++++++++++++++++
.../checklist.html | 262 ++---
.../codecs.html | 316 +++---
.../functions.html | 263 ++---
extension-guide/index.html | 873 +++++++++++++++++
.../other-components.html | 163 ++-
.../query-planners.html | 185 +++-
.../index.html => extension-guide/sessions.html | 206 +++-
.../table-providers.html | 294 +++---
.../why-ffi.html | 285 +++---
genindex.html | 304 +++++-
index.html | 54 +-
links.html | 41 +-
llms.txt | 14 +-
objects.inv | Bin 11426 -> 13365 bytes
py-modindex.html | 46 +-
search.html | 41 +-
searchindex.js | 2 +-
user-guide/ai-coding-assistants.html | 47 +-
user-guide/basics.html | 41 +-
user-guide/common-operations/aggregations.html | 70 +-
user-guide/common-operations/basic-info.html | 75 +-
user-guide/common-operations/expressions.html | 46 +-
user-guide/common-operations/functions.html | 92 +-
user-guide/common-operations/index.html | 43 +-
user-guide/common-operations/joins.html | 41 +-
.../common-operations/select-and-filter.html | 41 +-
user-guide/common-operations/spark-functions.html | 41 +-
user-guide/common-operations/udf-and-udfa.html | 48 +-
user-guide/common-operations/views.html | 41 +-
user-guide/common-operations/windows.html | 109 ++-
user-guide/configuration.html | 120 ++-
user-guide/data-sources.html | 78 +-
user-guide/dataframe/execution-metrics.html | 41 +-
user-guide/dataframe/index.html | 41 +-
user-guide/dataframe/rendering.html | 41 +-
user-guide/distributing-work.html | 928 +-----------------
.../expressions.html} | 339 +++----
.../arrow.html => distributing-work/index.html} | 218 +++--
.../query-engines.html} | 220 +++--
user-guide/{basics.html => extensions.html} | 249 +++--
user-guide/index.html | 65 +-
user-guide/introduction.html | 43 +-
user-guide/io/arrow.html | 41 +-
user-guide/io/avro.html | 41 +-
user-guide/io/csv.html | 41 +-
user-guide/io/index.html | 41 +-
user-guide/io/json.html | 41 +-
user-guide/io/parquet.html | 41 +-
user-guide/io/table_provider.html | 62 +-
user-guide/sql.html | 47 +-
user-guide/upgrade-guides.html | 426 +++++++-
128 files changed, 17423 insertions(+), 4347 deletions(-)
diff --git a/_sources/autoapi/datafusion/catalog/index.rst.txt b/_sources/autoapi/datafusion/catalog/index.rst.txt
index dcd21725..939d195a 100644
--- a/_sources/autoapi/datafusion/catalog/index.rst.txt
+++ b/_sources/autoapi/datafusion/catalog/index.rst.txt
@@ -21,6 +21,7 @@ Classes
datafusion.catalog.Schema
datafusion.catalog.SchemaProvider
datafusion.catalog.Table
+ datafusion.catalog.TableProviderFactory
Module Contents
@@ -376,3 +377,19 @@ Module Contents
Returns the schema associated with this table.
+.. py:class:: TableProviderFactory
+
+ Bases: :py:obj:`abc.ABC`
+
+
+ Abstract class for defining a Python based Table Provider Factory.
+
+
+ .. py:method:: create(cmd: datafusion.expr.CreateExternalTable) -> Table
+ :abstractmethod:
+
+
+ Create a table using the :class:`CreateExternalTable`.
+
+
+
diff --git a/_sources/autoapi/datafusion/context/index.rst.txt b/_sources/autoapi/datafusion/context/index.rst.txt
index ad379f02..25b88317 100644
--- a/_sources/autoapi/datafusion/context/index.rst.txt
+++ b/_sources/autoapi/datafusion/context/index.rst.txt
@@ -41,7 +41,6 @@ Classes
datafusion.context.ArrowArrayExportable
datafusion.context.ArrowStreamExportable
- datafusion.context.PhysicalOptimizerRuleExportable
datafusion.context.RuntimeEnvBuilder
datafusion.context.SQLOptions
datafusion.context.SessionConfig
@@ -78,20 +77,6 @@ Module Contents
.. py:method:: __arrow_c_stream__(requested_schema: object | None = None) -> object
-.. py:class:: PhysicalOptimizerRuleExportable
-
- Bases: :py:obj:`Protocol`
-
-
- Type hint for object that has __datafusion_physical_optimizer_rule__ PyCapsule.
-
- The method returns a PyCapsule wrapping an ``FFI_PhysicalOptimizerRule``,
- typically produced by a separate compiled extension.
-
-
- .. py:method:: __datafusion_physical_optimizer_rule__() -> object
-
-
.. py:class:: RuntimeEnvBuilder
Runtime configuration options.
@@ -264,18 +249,60 @@ Module Contents
Create a new :py:class:`SessionConfig` with the given configuration options.
- :param config_options: Configuration options.
+ Each entry is applied as though passed to :py:meth:`set`, so the same
+ keys are rejected. See :ref:`configuration`.
+
+ :param config_options: Options to apply, keyed by fully qualified name.
+
+ :raises ValueError: If a key names no known option, or a value does not
+ parse as that option's declared type. Which of several bad
+ entries is reported is not defined.
+
+ Example usage:
+
+ >>> from datafusion import SessionConfig, SessionContext
+ >>> ctx = SessionContext(SessionConfig())
+ >>> config = SessionConfig(
+ ... config_options={"datafusion.execution.batch_size": "1024"}
+ ... )
+ >>> ctx = SessionContext(config.with_information_schema(True))
+ >>> ctx.sql(
+ ... "select value from information_schema.df_settings"
+ ... " where name = 'datafusion.execution.batch_size'"
+ ... ).collect()[0]["value"][0]
+ <pyarrow.StringScalar: '1024'>
.. py:method:: set(key: str, value: str) -> SessionConfig
- Set a configuration option.
+ Set a configuration option by its fully qualified key.
- Args:
- key: Option key.
- value: Option value.
+ Not every key that ``information_schema.df_settings`` lists can be set
+ here: the ``datafusion.runtime.*`` entries come from the runtime
+ environment rather than from the session config. See
+ :ref:`configuration`.
- :returns: A new :py:class:`SessionConfig` object with the updated setting.
+ :param key: Option key including its namespace, such as
+ ``datafusion.execution.batch_size``.
+ :param value: Option value as a string, parsed according to the type the
+ option declares.
+
+ :returns: This :py:class:`SessionConfig`, modified in place, so that calls
+ chain.
+
+ :raises ValueError: If ``key`` names no known option, or if ``value`` does
+ not parse as that option's declared type.
+
+ Example usage:
+
+ >>> from datafusion import SessionConfig, SessionContext
+ >>> config = SessionConfig().set("datafusion.execution.batch_size", "1024")
+ >>> ctx = SessionContext(config.with_information_schema(True))
+ >>> ctx.sql(
+ ... "select value from information_schema.df_settings"
+ ... " where name = 'datafusion.execution.batch_size'"
+ ... ).collect()[0]["value"][0]
+ <pyarrow.StringScalar: '1024'>
@@ -431,6 +458,27 @@ Module Contents
See :ref:`user_guide_concepts` in the online documentation for more information.
+ **A context is a handle on a session, not the session itself.** The
+ ``with_*`` methods — :py:meth:`with_logical_extension_codec`,
+ :py:meth:`with_physical_extension_codec`,
+ :py:meth:`with_python_udf_inlining`, and :py:meth:`with_extensions` —
+ return a new context wrapping the *same* underlying session. Only the
+ Python-side codec settings differ; catalogs, tables, registered functions,
+ and configuration are the one shared session, so a registration through
+ either handle is visible to both.
+
+ A few things therefore belong to the session rather than to a handle, and
+ take effect even if the handle that set them is discarded: the query
+ planner (see :py:meth:`set_query_planner`), and the rebuild of an installed
+ foreign planner that follows installing a codec. :ref:`extension_sessions`
+ in the online documentation works through when that matters.
+
+ **Keep a context alive for as long as anything derived from it is in use.**
+ A :py:class:`~datafusion.DataFrame`, logical plan, or exported capsule does
+ not extend the session's lifetime. Once the last context on a session is
+ collected, any operation that reaches an extension codec fails with
+ ``TaskContextProvider went out of scope over FFI boundary``.
+
Main interface for executing queries with DataFusion.
Maintains the state of the connection between a user and an instance
@@ -451,16 +499,75 @@ Module Contents
df = ctx.read_csv("data.csv")
- .. py:method:: __datafusion_logical_extension_codec__() -> Any
+ .. py:method:: __datafusion_logical_extension_codec__(session: Any = None) -> Any
Access the PyCapsule FFI_LogicalExtensionCodec.
+ ``session`` is accepted so a context satisfies the same protocol an
+ extension library implements, where the argument is how the library
+ reaches the host's codec. A context already carries one, so the
+ argument is ignored. See
+ :py:class:`~datafusion.user_defined.LogicalExtensionCodecExportable`
+ for what an extension library does with it.
+ :param session: Accepted and ignored.
- .. py:method:: __datafusion_physical_extension_codec__() -> Any
+ :returns: A ``datafusion_logical_extension_codec`` PyCapsule.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> type(SessionContext().__datafusion_logical_extension_codec__()).__name__
+ 'PyCapsule'
+
+
+
+ .. py:method:: __datafusion_physical_extension_codec__(session: Any = None) -> Any
Access the PyCapsule FFI_PhysicalExtensionCodec.
+ See :meth:`__datafusion_logical_extension_codec__` for ``session``.
+
+ :param session: Accepted and ignored.
+
+ :returns: A ``datafusion_physical_extension_codec`` PyCapsule.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> type(ctx.__datafusion_physical_extension_codec__()).__name__
+ 'PyCapsule'
+
+
+
+ .. py:method:: __datafusion_query_planner__(session: Any = None) -> Any
+
+ Access the ``FFI_QueryPlanner`` PyCapsule for the current planner.
+
+ This is how you capture the planner a session already has in order to
+ wrap it. Capture it *before* installing the new one, since
+ :py:meth:`set_query_planner` replaces rather than layers.
+
+ :param session: Accepted and ignored. See
+ :meth:`__datafusion_logical_extension_codec__`.
+
+ :returns: A ``datafusion_query_planner`` PyCapsule wrapping the session's
+ current planner, exported for a foreign planner to delegate to.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> fallback = ctx.__datafusion_query_planner__()
+ >>> type(fallback).__name__
+ 'PyCapsule'
+
+ Both spellings work, since the argument is ignored:
+
+ >>> type(ctx.__datafusion_query_planner__(ctx)).__name__
+ 'PyCapsule'
+
.. py:method:: __datafusion_task_context_provider__() -> Any
@@ -492,7 +599,25 @@ Module Contents
- .. py:method:: add_physical_optimizer_rule(rule: PhysicalOptimizerRuleExportable) -> None
+ .. py:method:: _register_object_store_for_path(path: str | pathlib.Path, store: Any) -> None
+
+ Parse a URL path and register the given object store for its scheme and host.
+
+ This is a convenience helper used by methods like
+ :py:meth:`register_parquet` and :py:meth:`read_parquet` to
+ automatically register an object store when an ``object_store``
+ parameter is provided.
+
+ :param path: A URL-style path (e.g. ``"s3://bucket/key.parquet"`` or
+ ``"file:///tmp/data.parquet"``).
+ :param store: An object store instance to register.
+
+ :raises ValueError: If the path does not contain a URL scheme, or if
+ a non-file scheme is missing a host/bucket component.
+
+
+
+ .. py:method:: add_physical_optimizer_rule(rule: datafusion.extensions.PhysicalOptimizerRuleExportable) -> None
Append a user-defined physical optimizer rule to the session.
@@ -503,7 +628,8 @@ Module Contents
and catalogs are preserved.
:param rule: Object exposing ``__datafusion_physical_optimizer_rule__``,
- a :class:`PhysicalOptimizerRuleExportable`.
+ a
+ :py:class:`~datafusion.extensions.PhysicalOptimizerRuleExportable`.
.. rubric:: Examples
@@ -663,9 +789,37 @@ Module Contents
- .. py:method:: execute(plan: datafusion.plan.ExecutionPlan, partitions: int) -> datafusion.record_batch.RecordBatchStream
+ .. py:method:: execute(plan: datafusion.plan.ExecutionPlan, partition: int) -> datafusion.record_batch.RecordBatchStream
+
+ Execute a single partition of ``plan`` and stream its batches.
+
+ :param plan: The physical plan to execute.
+ :param partition: Index of the partition to execute, in
+ ``range(plan.partition_count)``.
- Execute the ``plan`` and return the results.
+ :returns: A stream over the record batches that partition produces.
+
+ :raises ValueError: If ``partition`` is not a valid index for ``plan``.
+ :raises OverflowError: If ``partition`` is negative, or too large to fit a
+ platform-sized unsigned integer.
+
+ Example usage:
+
+ >>> import pyarrow as pa
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> ctx.register_record_batches(
+ ... "t", [[pa.record_batch({"a": [1, 2]})], [pa.record_batch({"a": [3]})]]
+ ... )
+ >>> plan = ctx.sql("select a from t").execution_plan()
+ >>> plan.partition_count
+ 2
+ >>> sum(
+ ... batch.to_pyarrow().num_rows
+ ... for p in range(plan.partition_count)
+ ... for batch in ctx.execute(plan, p)
+ ... )
+ 3
@@ -764,6 +918,36 @@ Module Contents
+ .. py:method:: logical_extension_codec_ids() -> list[str]
+
+ List the logical extension codecs installed on this session.
+
+ Returns the identity of each installed codec, in install order. Those
+ identities are what encoding stamps onto a payload and what decoding
+ dispatches on, so this is how to check which library owns a plan and
+ whether a session is able to decode one.
+
+ DataFusion's own default codec is not listed. It handles whatever no
+ installed codec claims, and it carries no identity to list.
+
+ :returns: The installed codec ids, in install order. Empty if none are
+ installed.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> host = SessionContext()
+ >>> host.logical_extension_codec_ids()
+ []
+ >>> capsule = host.__datafusion_logical_extension_codec__()
+ >>> ctx = SessionContext().with_logical_extension_codec(
+ ... capsule, codec_id="my_library.Codec"
+ ... )
+ >>> ctx.logical_extension_codec_ids()
+ ['my_library.Codec']
+
+
+
.. py:method:: parse_capacity_limit(config_name: str, limit: str) -> int
:staticmethod:
@@ -808,7 +992,31 @@ Module Contents
- .. py:method:: read_arrow(path: str | pathlib.Path, schema: pyarrow.Schema | None = None, file_extension: str = '.arrow', file_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None) -> datafusion.dataframe.DataFrame
+ .. py:method:: physical_extension_codec_ids() -> list[str]
+
+ List the physical extension codecs installed on this session.
+
+ See :py:meth:`logical_extension_codec_ids`.
+
+ :returns: The installed codec ids, in install order. Empty if none are
+ installed.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> host = SessionContext()
+ >>> host.physical_extension_codec_ids()
+ []
+ >>> capsule = host.__datafusion_physical_extension_codec__()
+ >>> ctx = SessionContext().with_physical_extension_codec(
+ ... capsule, codec_id="my_library.PhysicalCodec"
+ ... )
+ >>> ctx.physical_extension_codec_ids()
+ ['my_library.PhysicalCodec']
+
+
+
+ .. py:method:: read_arrow(path: str | pathlib.Path, schema: pyarrow.Schema | None = None, file_extension: str = '.arrow', file_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, object_store: Any | None = None) -> datafusion.dataframe.DataFrame
Create a :py:class:`DataFrame` for reading an Arrow IPC data source.
@@ -816,6 +1024,9 @@ Module Contents
:param schema: The data source schema.
:param file_extension: File extension to select.
:param file_partition_cols: Partition columns.
+ :param object_store: A pre-configured object store instance to use for
+ accessing the file. When provided, the store is automatically
+ registered for the URL scheme and host parsed from ``path``.
:returns: DataFrame representation of the read Arrow IPC file.
@@ -869,7 +1080,7 @@ Module Contents
- .. py:method:: read_avro(path: str | pathlib.Path, schema: pyarrow.Schema | None = None, file_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, file_extension: str = '.avro') -> datafusion.dataframe.DataFrame
+ .. py:method:: read_avro(path: str | pathlib.Path, schema: pyarrow.Schema | None = None, file_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, file_extension: str = '.avro', object_store: Any | None = None) -> datafusion.dataframe.DataFrame
Create a :py:class:`DataFrame` for reading Avro data source.
@@ -877,6 +1088,9 @@ Module Contents
:param schema: The data source schema.
:param file_partition_cols: Partition columns.
:param file_extension: File extension to select.
+ :param object_store: A pre-configured object store instance to use for
+ accessing the file. When provided, the store is automatically
+ registered for the URL scheme and host parsed from ``path``.
:returns: DataFrame representation of the read Avro file
@@ -930,7 +1144,7 @@ Module Contents
- .. py:method:: read_csv(path: str | pathlib.Path | list[str] | list[pathlib.Path], schema: pyarrow.Schema | None = None, has_header: bool = True, delimiter: str = ',', schema_infer_max_records: int = DEFAULT_MAX_INFER_SCHEMA, file_extension: str = '.csv', table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, file_compression_type: str | None = None, options: datafusion.options.CsvReadOptions | None = None) -> datafusion.dataframe.DataFrame
+ .. py:method:: read_csv(path: str | pathlib.Path | list[str] | list[pathlib.Path], schema: pyarrow.Schema | None = None, has_header: bool = True, delimiter: str = ',', schema_infer_max_records: int = DEFAULT_MAX_INFER_SCHEMA, file_extension: str = '.csv', table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, file_compression_type: str | None = None, options: datafusion.options.CsvReadOptions | None = None, object_store: Any | None = None) -> datafusion.datafram [...]
Read a CSV data source.
@@ -949,6 +1163,9 @@ Module Contents
:param file_compression_type: File compression type.
:param options: Set advanced options for CSV reading. This cannot be
combined with any of the other options in this method.
+ :param object_store: A pre-configured object store instance to use for
+ accessing the file. When provided, the store is automatically
+ registered for the URL scheme and host parsed from ``path``.
:returns: DataFrame representation of the read CSV files
@@ -962,7 +1179,7 @@ Module Contents
- .. py:method:: read_json(path: str | pathlib.Path, schema: pyarrow.Schema | None = None, schema_infer_max_records: int = 1000, file_extension: str = '.json', table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, file_compression_type: str | None = None) -> datafusion.dataframe.DataFrame
+ .. py:method:: read_json(path: str | pathlib.Path, schema: pyarrow.Schema | None = None, schema_infer_max_records: int = 1000, file_extension: str = '.json', table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, file_compression_type: str | None = None, object_store: Any | None = None) -> datafusion.dataframe.DataFrame
Read a line-delimited JSON data source.
@@ -974,12 +1191,15 @@ Module Contents
selected for data input.
:param table_partition_cols: Partition columns.
:param file_compression_type: File compression type.
+ :param object_store: A pre-configured object store instance to use for
+ accessing the file. When provided, the store is automatically
+ registered for the URL scheme and host parsed from ``path``.
:returns: DataFrame representation of the read JSON files.
- .. py:method:: read_parquet(path: str | pathlib.Path, table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, parquet_pruning: bool = True, file_extension: str = '.parquet', skip_metadata: bool = True, schema: pyarrow.Schema | None = None, file_sort_order: collections.abc.Sequence[collections.abc.Sequence[datafusion.expr.SortKey]] | None = None) -> datafusion.dataframe.DataFrame
+ .. py:method:: read_parquet(path: str | pathlib.Path, table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, parquet_pruning: bool = True, file_extension: str = '.parquet', skip_metadata: bool = True, schema: pyarrow.Schema | None = None, file_sort_order: collections.abc.Sequence[collections.abc.Sequence[datafusion.expr.SortKey]] | None = None, object_store: Any | None = None) -> datafusion.dataframe.DataFrame
Read a Parquet source into a :py:class:`~datafusion.dataframe.Dataframe`.
@@ -998,9 +1218,41 @@ Module Contents
:param file_sort_order: Sort order for the file. Each sort key can be
specified as a column name (``str``), an expression
(``Expr``), or a ``SortExpr``.
+ :param object_store: A pre-configured object store instance (e.g.
+ :py:class:`~datafusion.object_store.AmazonS3`,
+ :py:class:`~datafusion.object_store.GoogleCloud`,
+ :py:class:`~datafusion.object_store.MicrosoftAzure`) to use
+ for accessing the file. When provided, the store is
+ automatically registered for the URL scheme and host parsed
+ from ``path``, removing the need to call
+ :py:meth:`register_object_store` separately. This is
+ especially useful in multi-threaded environments where
+ setting credentials via ``os.environ`` is not thread-safe.
:returns: DataFrame representation of the read Parquet files
+ .. rubric:: Examples
+
+ Read a local Parquet file:
+
+ >>> import datafusion
+ >>> ctx = datafusion.SessionContext()
+ >>> df = ctx.read_parquet("data.parquet") # doctest: +SKIP
+
+ Read from S3 with inline credentials (thread-safe):
+
+ >>> from datafusion.object_store import AmazonS3 # doctest: +SKIP
+ >>> store = AmazonS3(
+ ... bucket_name="my-bucket",
+ ... region="us-east-1",
+ ... access_key_id="...",
+ ... secret_access_key="...",
+ ... ) # doctest: +SKIP
+ >>> df = ctx.read_parquet(
+ ... "s3://my-bucket/data.parquet",
+ ... object_store=store,
+ ... ) # doctest: +SKIP
+
.. py:method:: read_table(table: datafusion.catalog.Table | TableProviderExportable | datafusion.dataframe.DataFrame | pyarrow.dataset.Dataset) -> datafusion.dataframe.DataFrame
@@ -1020,7 +1272,7 @@ Module Contents
- .. py:method:: register_arrow(name: str, path: str | pathlib.Path, schema: pyarrow.Schema | None = None, file_extension: str = '.arrow', table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None) -> None
+ .. py:method:: register_arrow(name: str, path: str | pathlib.Path, schema: pyarrow.Schema | None = None, file_extension: str = '.arrow', table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, object_store: Any | None = None) -> None
Register an Arrow IPC file as a table.
@@ -1032,6 +1284,9 @@ Module Contents
:param schema: The data source schema.
:param file_extension: File extension to select.
:param table_partition_cols: Partition columns.
+ :param object_store: A pre-configured object store instance to use for
+ accessing the file. When provided, the store is automatically
+ registered for the URL scheme and host parsed from ``path``.
.. rubric:: Examples
@@ -1089,7 +1344,7 @@ Module Contents
- .. py:method:: register_avro(name: str, path: str | pathlib.Path, schema: pyarrow.Schema | None = None, file_extension: str = '.avro', table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None) -> None
+ .. py:method:: register_avro(name: str, path: str | pathlib.Path, schema: pyarrow.Schema | None = None, file_extension: str = '.avro', table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, object_store: Any | None = None) -> None
Register an Avro file as a table.
@@ -1101,6 +1356,9 @@ Module Contents
:param schema: The data source schema.
:param file_extension: File extension to select.
:param table_partition_cols: Partition columns.
+ :param object_store: A pre-configured object store instance to use for
+ accessing the file. When provided, the store is automatically
+ registered for the URL scheme and host parsed from ``path``.
@@ -1138,7 +1396,7 @@ Module Contents
- .. py:method:: register_csv(name: str, path: str | pathlib.Path | list[str | pathlib.Path], schema: pyarrow.Schema | None = None, has_header: bool = True, delimiter: str = ',', schema_infer_max_records: int = DEFAULT_MAX_INFER_SCHEMA, file_extension: str = '.csv', file_compression_type: str | None = None, options: datafusion.options.CsvReadOptions | None = None) -> None
+ .. py:method:: register_csv(name: str, path: str | pathlib.Path | list[str | pathlib.Path], schema: pyarrow.Schema | None = None, has_header: bool = True, delimiter: str = ',', schema_infer_max_records: int = DEFAULT_MAX_INFER_SCHEMA, file_extension: str = '.csv', file_compression_type: str | None = None, options: datafusion.options.CsvReadOptions | None = None, object_store: Any | None = None) -> None
Register a CSV file as a table.
@@ -1159,6 +1417,9 @@ Module Contents
:param file_compression_type: File compression type.
:param options: Set advanced options for CSV reading. This cannot be
combined with any of the other options in this method.
+ :param object_store: A pre-configured object store instance to use for
+ accessing the file. When provided, the store is automatically
+ registered for the URL scheme and host parsed from ``path``.
@@ -1171,7 +1432,7 @@ Module Contents
- .. py:method:: register_json(name: str, path: str | pathlib.Path, schema: pyarrow.Schema | None = None, schema_infer_max_records: int = 1000, file_extension: str = '.json', table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, file_compression_type: str | None = None) -> None
+ .. py:method:: register_json(name: str, path: str | pathlib.Path, schema: pyarrow.Schema | None = None, schema_infer_max_records: int = 1000, file_extension: str = '.json', table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, file_compression_type: str | None = None, object_store: Any | None = None) -> None
Register a JSON file as a table.
@@ -1187,6 +1448,9 @@ Module Contents
selected for data input.
:param table_partition_cols: Partition columns.
:param file_compression_type: File compression type.
+ :param object_store: A pre-configured object store instance to use for
+ accessing the file. When provided, the store is automatically
+ registered for the URL scheme and host parsed from ``path``.
@@ -1219,7 +1483,7 @@ Module Contents
- .. py:method:: register_parquet(name: str, path: str | pathlib.Path, table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, parquet_pruning: bool = True, file_extension: str = '.parquet', skip_metadata: bool = True, schema: pyarrow.Schema | None = None, file_sort_order: collections.abc.Sequence[collections.abc.Sequence[datafusion.expr.SortKey]] | None = None) -> None
+ .. py:method:: register_parquet(name: str, path: str | pathlib.Path, table_partition_cols: list[tuple[str, str | pyarrow.DataType]] | None = None, parquet_pruning: bool = True, file_extension: str = '.parquet', skip_metadata: bool = True, schema: pyarrow.Schema | None = None, file_sort_order: collections.abc.Sequence[collections.abc.Sequence[datafusion.expr.SortKey]] | None = None, object_store: Any | None = None) -> None
Register a Parquet file as a table.
@@ -1240,6 +1504,39 @@ Module Contents
:param file_sort_order: Sort order for the file. Each sort key can be
specified as a column name (``str``), an expression
(``Expr``), or a ``SortExpr``.
+ :param object_store: A pre-configured object store instance (e.g.
+ :py:class:`~datafusion.object_store.AmazonS3`,
+ :py:class:`~datafusion.object_store.GoogleCloud`,
+ :py:class:`~datafusion.object_store.MicrosoftAzure`) to use
+ for accessing the file. When provided, the store is
+ automatically registered for the URL scheme and host parsed
+ from ``path``, removing the need to call
+ :py:meth:`register_object_store` separately. This is
+ especially useful in multi-threaded environments where
+ setting credentials via ``os.environ`` is not thread-safe.
+
+ .. rubric:: Examples
+
+ Register a local Parquet file:
+
+ >>> import datafusion
+ >>> ctx = datafusion.SessionContext()
+ >>> ctx.register_parquet("my_table", "data.parquet") # doctest: +SKIP
+
+ Register from S3 with inline credentials (thread-safe):
+
+ >>> from datafusion.object_store import AmazonS3 # doctest: +SKIP
+ >>> store = AmazonS3(
+ ... bucket_name="my-bucket",
+ ... region="us-east-1",
+ ... access_key_id="...",
+ ... secret_access_key="...",
+ ... ) # doctest: +SKIP
+ >>> ctx.register_parquet(
+ ... "my_table",
+ ... "s3://my-bucket/data.parquet",
+ ... object_store=store,
+ ... ) # doctest: +SKIP
@@ -1356,6 +1653,57 @@ Module Contents
+ .. py:method:: set_query_planner(planner: datafusion.extensions.QueryPlannerExportable | types.CapsuleType) -> None
+
+ Install a custom query planner on this session.
+
+ The planner is imported through its ``__datafusion_query_planner__``
+ PyCapsule, in the same way
+ :meth:`~SessionContext.add_physical_optimizer_rule` installs a rule.
+
+ Returns nothing, because the planner lives in the session rather than
+ in a handle on it — installing one is visible to every context sharing
+ that session, including ones an earlier ``with_*`` call returned. See
+ :py:class:`SessionContext`.
+
+ A session holds exactly one planner, so calling this again **replaces**
+ the previous one rather than layering. To chain planners, have the new
+ planner wrap the capsule from
+ :meth:`~SessionContext.__datafusion_query_planner__`, captured *before*
+ the new planner is installed.
+
+ Install any extension codecs before a layered planner; the rebuild that
+ follows a later codec install does not reach the fallback inside one.
+ See :ref:`planner_codec_rebinding`, or prefer
+ :py:meth:`with_extensions`, which cannot capture a partial chain.
+
+ :param planner: Object exposing ``__datafusion_query_planner__`` (see
+ :py:class:`~datafusion.extensions.QueryPlannerExportable`) or a raw
+ ``datafusion_query_planner`` PyCapsule.
+
+ :raises ValueError: If the capsule is not named ``datafusion_query_planner``.
+
+ .. rubric:: Examples
+
+ A session exports its own planner, which is what you capture to
+ wrap:
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> fallback = ctx.__datafusion_query_planner__()
+ >>> type(fallback).__name__
+ 'PyCapsule'
+
+ Skipped here (needs a built extension library):
+
+ >>> from my_extension import DistributedQueryPlanner # doctest: +SKIP
+ >>> ctx.set_query_planner(
+ ... DistributedQueryPlanner(fallback=fallback)
+ ... ) # doctest: +SKIP
+ >>> ctx.sql("SELECT * FROM remote_table").collect() # doctest: +SKIP
+
+
+
.. py:method:: sql(query: str, options: SQLOptions | None = None, param_values: dict[str, Any] | None = None, **named_params: Any) -> datafusion.dataframe.DataFrame
Create a :py:class:`~datafusion.DataFrame` from SQL query text.
@@ -1567,34 +1915,190 @@ Module Contents
- .. py:method:: with_logical_extension_codec(codec: datafusion.user_defined.LogicalExtensionCodecExportable | _typeshed.CapsuleType) -> SessionContext
+ .. py:method:: with_extensions(*extensions: datafusion.extensions.SessionComponentsExportable | datafusion.extensions.SessionPlannerExportable) -> SessionContext
+
+ Create a new session context with the given extension bundles.
+
+ This is the preferred way to install extension codecs and query
+ planners, because it removes the ordering question that installing them
+ by hand creates.
+
+ Each argument is called twice, in two phases:
+
+ 1. ``__datafusion_session_components__(ctx)`` on every extension, then
+ all the returned codecs are installed at once.
+ 2. ``__datafusion_session_planner__(ctx, fallback)`` on every
+ extension, **in argument order**, each handed the planner built so
+ far and the context carrying every bundle's codecs. The last
+ extension listed ends up outermost.
+
+ An extension implements either hook or both. Return ``None`` from the
+ planner hook to contribute no planner; see
+ :py:class:`~datafusion.extensions.SessionPlannerExportable`.
+
+ Nothing is written to the session until every hook has returned and
+ every capsule has been validated, so a hook that raises leaves the
+ session as it was. A hook that *mutates* the context it is handed —
+ registering a table, say — is not rolled back, which is why bundle
+ objects must be configuration-only. See
+ :ref:`extension_bundles_transaction`.
+
+ Shares its session with this context — see :py:class:`SessionContext`.
+
+ See :ref:`extension_bundles` in the online documentation for why the
+ phases are split, how to contribute a bundle's two halves at different
+ positions, and a worked Rust implementation.
+
+ :param extensions: Extension bundles to install. Order is irrelevant for
+ codecs and significant for planners, which nest in this order
+ with the last one outermost. Passing none installs nothing and
+ returns a handle on this session, so a caller assembling the
+ list at runtime need not special-case it being empty.
+
+ :returns: A new context with all extension components installed.
+
+ :raises TypeError: If an argument implements neither hook, if a hook
+ returns the wrong type, or if a codec is contributed as a bare
+ ``PyCapsule`` rather than an object exposing the getter.
+ :raises ValueError: If two codecs claim the same id, or a getter returns a
+ capsule of the wrong kind. See
+ :py:meth:`with_logical_extension_codec` for how ids are
+ assigned.
+
+ .. rubric:: Examples
+
+ The returned handle is a different object sharing one session, and
+ an empty call is legal:
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> derived = ctx.with_extensions()
+ >>> derived is ctx
+ False
+ >>> ctx.from_pydict({"a": [1, 2]}, name="t") # doctest: +ELLIPSIS
+ DataFrame()...
+ >>> derived.table_exist("t")
+ True
+ >>> derived.logical_extension_codec_ids()
+ []
+
+ A runnable multi-bundle example, showing what a bundle returns and
+ how its codec ids accumulate, is in :ref:`extension_bundles`.
+
+ Real usage. Skipped here (needs a built extension library); run
+ verbatim by ``test_with_extensions_docstring_example_still_runs``.
- Create a new session context with specified codec.
+ >>> from my_extension import DistributedEngineExtension # doctest: +SKIP
+ >>> ctx = SessionContext().with_extensions(
+ ... DistributedEngineExtension("scheduler:50050")
+ ... ) # doctest: +SKIP
+ >>> batches = ctx.sql("SELECT 1 AS n").collect() # doctest: +SKIP
+ >>> batches[0].column(0).to_pylist() # doctest: +SKIP
+ [1]
+
+
+
+ .. py:method:: with_logical_extension_codec(codec: datafusion.user_defined.LogicalExtensionCodecExportable | types.CapsuleType, codec_id: str | None = None) -> SessionContext
+
+ Create a new session context with an additional logical codec.
Only FFI codecs are supported. Pass any object implementing
``__datafusion_logical_extension_codec__`` (see
:py:class:`~datafusion.user_defined.LogicalExtensionCodecExportable`).
+ Codecs compose: each call appends the codec rather than replacing
+ codecs installed earlier, so one session can carry codecs from several
+ independent libraries and the order they are installed in does not
+ affect decoding.
+
+ A serialized plan records which codec wrote each payload, as a short id
+ taken from the codec's class.
+
+ Shares its session with this context — see :py:class:`SessionContext`.
+ See :ref:`extension_codec_ids` in the online documentation for how ids
+ are assigned and what an extension codec has to implement, and
+ :py:meth:`with_extensions` for installing a library's codecs and planner
+ together.
- .. py:method:: with_physical_extension_codec(codec: datafusion.user_defined.PhysicalExtensionCodecExportable | _typeshed.CapsuleType) -> SessionContext
+ :param codec: Object implementing ``__datafusion_logical_extension_codec__``
+ (see
+ :py:class:`~datafusion.user_defined.LogicalExtensionCodecExportable`),
+ or a raw ``datafusion_logical_extension_codec`` PyCapsule.
+ :param codec_id: Overrides the id the codec's payloads are tagged with.
+ Normally unnecessary. Pass it when installing two instances of
+ one class, which otherwise claim the same id, and when
+ installing a bare ``PyCapsule``: a capsule has no class to take
+ an id from, so it is given a random ``anon:`` id that differs on
+ every install, and plans it encodes can never be decoded
+ elsewhere.
- Create a new session context with the specified physical codec.
+ :returns: A new context carrying this codec in addition to any already
+ installed.
+
+ :raises ValueError: If the resolved id is already installed on this session.
+
+ .. rubric:: Examples
+
+ A context exports its own codec, which stands in here for a real
+ library's:
+
+ >>> from datafusion import SessionContext
+ >>> host = SessionContext()
+ >>> capsule = host.__datafusion_logical_extension_codec__()
+ >>> ctx = SessionContext().with_logical_extension_codec(
+ ... capsule, codec_id="my_library.Codec"
+ ... )
+ >>> ctx.logical_extension_codec_ids()
+ ['my_library.Codec']
+
+ Without ``codec_id`` a bare capsule gets an anonymous id, which is
+ fine only if its plans never leave this session:
+
+ >>> ctx = SessionContext().with_logical_extension_codec(capsule)
+ >>> ctx.logical_extension_codec_ids()[0].startswith("anon:")
+ True
+
+
+
+ .. py:method:: with_physical_extension_codec(codec: datafusion.user_defined.PhysicalExtensionCodecExportable | types.CapsuleType, codec_id: str | None = None) -> SessionContext
+
+ Create a new session context with an additional physical codec.
Only FFI codecs are supported. Pass any object implementing
``__datafusion_physical_extension_codec__`` (see
:py:class:`~datafusion.user_defined.PhysicalExtensionCodecExportable`).
+ Composes and assigns an id exactly as
+ :py:meth:`with_logical_extension_codec` does, including when to pass
+ ``codec_id`` and what the returned context shares. See that method.
+
+ :param codec: As :py:meth:`with_logical_extension_codec`, for the physical
+ getter.
+ :param codec_id: As :py:meth:`with_logical_extension_codec`.
+
+ :returns: A new context carrying this codec in addition to any already
+ installed.
+
+ :raises ValueError: If the resolved id is already installed on this session.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> host = SessionContext()
+ >>> capsule = host.__datafusion_physical_extension_codec__()
+ >>> ctx = SessionContext().with_physical_extension_codec(
+ ... capsule, codec_id="my_library.PhysicalCodec"
+ ... )
+ >>> ctx.physical_extension_codec_ids()
+ ['my_library.PhysicalCodec']
+
.. py:method:: with_python_udf_inlining(*, enabled: bool) -> SessionContext
Control whether Python UDFs are embedded in serialized expressions.
- ``enabled`` is keyword-only and required: callers must pick a
- mode explicitly. Fresh sessions inline UDFs (``enabled=True``
- behavior) until this method overrides the toggle.
-
With ``enabled=True``, serialized expressions carry the Python
code for any scalar, aggregate, or window UDFs they reference.
The receiver rebuilds the UDFs from those bytes and does not
@@ -1623,8 +2127,16 @@ Module Contents
:func:`pickle.loads` on untrusted bytes remains unsafe
regardless of the toggle.
- Returns a new :class:`SessionContext` with the toggle applied;
- the original session is unchanged.
+ Shares its session with this context — see
+ :py:class:`SessionContext`. The original context's own codec
+ settings are unchanged.
+
+ :param enabled: Whether to embed Python UDFs in serialized
+ expressions. Keyword-only and required, so callers must
+ pick a mode explicitly. Fresh sessions behave as
+ ``enabled=True`` until this method overrides the toggle.
+
+ :returns: A new :class:`SessionContext` with the toggle applied.
.. rubric:: Examples
@@ -1644,6 +2156,32 @@ Module Contents
+ .. py:property:: __datafusion_codec_id__
+ :type: str
+
+
+ Identity this context carries when installed as an extension codec.
+
+ A context can be installed on another session as an extension codec,
+ which tags the payloads it writes with this string. It is unique per
+ session, so two contexts can be installed on one session and a plan
+ written through one will not be decoded by the other.
+
+ Contexts derived from the same session report the same id, so only one
+ of them can be installed on a given session. That is the intended
+ answer: they are one session — see :py:class:`SessionContext` — so
+ their payloads would be indistinguishable on decode.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> ctx.__datafusion_codec_id__.startswith("session:")
+ True
+ >>> ctx.__datafusion_codec_id__ == SessionContext().__datafusion_codec_id__
+ False
+
+
.. py:attribute:: ctx
@@ -1654,7 +2192,16 @@ Module Contents
Type hint for object that has __datafusion_table_provider__ PyCapsule.
- https://datafusion.apache.org/python/user-guide/io/table_provider.html
+ See :ref:`io_custom_table_provider` for registering one, and
+ :ref:`extension_providers` for writing one.
+
+ :param session: See
+ :py:class:`~datafusion.user_defined.LogicalExtensionCodecExportable`.
+ For this getter it is a session when the provider is registered
+ through :py:meth:`SessionContext.register_table` and the host's
+ logical codec when it goes through
+ :py:meth:`datafusion.catalog.Schema.register_table`, so
+ duck-typing it is not optional.
.. py:method:: __datafusion_table_provider__(session: Any) -> object
diff --git a/_sources/autoapi/datafusion/dataframe/index.rst.txt b/_sources/autoapi/datafusion/dataframe/index.rst.txt
index bdc9fd78..3f737cd5 100644
--- a/_sources/autoapi/datafusion/dataframe/index.rst.txt
+++ b/_sources/autoapi/datafusion/dataframe/index.rst.txt
@@ -43,7 +43,9 @@ Classes
datafusion.dataframe.Compression
datafusion.dataframe.DataFrame
datafusion.dataframe.DataFrameWriteOptions
+ datafusion.dataframe.ExplainAnalyzeLevel
datafusion.dataframe.ExplainFormat
+ datafusion.dataframe.ExplainMetricCategory
datafusion.dataframe.InsertOp
datafusion.dataframe.ParquetColumnOptions
datafusion.dataframe.ParquetWriterOptions
@@ -524,7 +526,7 @@ Module Contents
- .. py:method:: explain(verbose: bool = False, analyze: bool = False, format: ExplainFormat | None = None) -> None
+ .. py:method:: explain(verbose: bool = False, analyze: bool = False, format: ExplainFormat | None = None, show_statistics: bool | None = None, analyze_level: ExplainAnalyzeLevel | None = None, analyze_categories: collections.abc.Iterable[ExplainMetricCategory] | None = None) -> None
Print an explanation of the DataFrame's plan so far.
@@ -534,6 +536,17 @@ Module Contents
:param analyze: If ``True``, the plan will run and metrics reported.
:param format: Output format for the plan. Defaults to
:py:attr:`ExplainFormat.INDENT`.
+ :param show_statistics: If ``True``, include each operator's statistics.
+ ``None`` uses the ``datafusion.explain.show_statistics``
+ setting.
+ :param analyze_level: Which metrics to report with ``analyze``. ``None``
+ uses the ``datafusion.explain.analyze_level`` setting.
+ :param analyze_categories: Report only metrics in these categories with
+ ``analyze``; an empty iterable reports none. ``None`` uses the
+ ``datafusion.explain.analyze_categories`` setting.
+
+ :raises ValueError: If ``show_statistics`` is set with ``analyze``, or
+ ``analyze_level`` or ``analyze_categories`` is set without it.
.. rubric:: Examples
@@ -548,6 +561,45 @@ Module Contents
>>> df.explain(analyze=True) # doctest: +SKIP
+ Show only row-count metrics:
+
+ >>> from datafusion.dataframe import ExplainMetricCategory
+ >>> df.explain(
+ ... analyze=True, analyze_categories=[ExplainMetricCategory.ROWS]
+ ... ) # doctest: +SKIP
+
+
+
+ .. py:method:: fill_nan(value: float, subset: list[str] | None = None) -> DataFrame
+
+ Fill NaN values in floating-point columns with a value.
+
+ Only floating-point columns are changed; others are kept unchanged, as is
+ any column ``value`` cannot be cast to. NaN is distinct from null, which
+ :py:meth:`fill_null` handles. Fails on a DataFrame with an uppercase or
+ dotted column name; see :ref:`fill_column_names`.
+
+ :param value: Value to replace NaN with. Will be cast to match column type.
+ :param subset: Optional list of column names to fill. If None, fills all
+ floating-point columns; an empty list fills none.
+
+ :returns: DataFrame with NaN values replaced.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> nan = float("nan")
+ >>> df = ctx.from_pydict({"a": [1.0, nan, None], "b": [nan, 2.0, 3.0]})
+ >>> df.fill_nan(0.0).to_pydict()
+ {'a': [1.0, 0.0, None], 'b': [0.0, 2.0, 3.0]}
+
+ >>> df.fill_nan(0.0, subset=["a"]).collect_column("b")[0].as_py()
+ nan
+
+ >>> df.fill_nan(0.0, subset=[]).collect_column("b")[0].as_py()
+ nan
+
.. py:method:: fill_null(value: Any, subset: list[str] | None = None) -> DataFrame
@@ -555,7 +607,8 @@ Module Contents
Fill null values in specified columns with a value.
:param value: Value to replace nulls with. Will be cast to match column type.
- :param subset: Optional list of column names to fill. If None, fills all columns.
+ :param subset: Optional list of column names to fill. If None, fills all columns;
+ an empty list fills none.
:returns: DataFrame with null values replaced where type casting is possible
@@ -568,11 +621,16 @@ Module Contents
>>> filled.sort(col("a")).collect()[0].column("a").to_pylist()
[0, 1, 3]
+ >>> df.fill_null(0, subset=[]).to_pydict()
+ {'a': [1, None, 3], 'b': [None, 5, 6]}
+
.. rubric:: Notes
- Only fills nulls in columns where the value can be cast to the column type
- For columns where casting fails, the original column is kept unchanged
- For columns not in subset, the original column is kept unchanged
+ - Fails on a DataFrame with an uppercase or dotted column name; see
+ :ref:`fill_column_names`
@@ -1345,6 +1403,28 @@ Module Contents
.. py:attribute:: _raw_write_options
+.. py:class:: ExplainAnalyzeLevel
+
+ Bases: :py:obj:`enum.Enum`
+
+
+ Which metrics :py:meth:`DataFrame.explain` reports when ``analyze=True``.
+
+
+ .. py:attribute:: DEV
+ :value: 'dev'
+
+
+ All metrics, including those for deep operator-level introspection.
+
+
+ .. py:attribute:: SUMMARY
+ :value: 'summary'
+
+
+ Common metrics for finding which operator is slow.
+
+
.. py:class:: ExplainFormat
Bases: :py:obj:`enum.Enum`
@@ -1383,6 +1463,42 @@ Module Contents
Tree-style visual format with box-drawing characters.
+.. py:class:: ExplainMetricCategory
+
+ Bases: :py:obj:`enum.Enum`
+
+
+ Category of metric reported by :py:meth:`DataFrame.explain` with ``analyze``.
+
+
+ .. py:attribute:: BYTES
+ :value: 'bytes'
+
+
+ Byte sizes, such as ``output_bytes``.
+
+
+ .. py:attribute:: ROWS
+ :value: 'rows'
+
+
+ Row counts, such as ``output_rows``.
+
+
+ .. py:attribute:: TIMING
+ :value: 'timing'
+
+
+ Elapsed times, such as ``elapsed_compute``.
+
+
+ .. py:attribute:: UNCATEGORIZED
+ :value: 'uncategorized'
+
+
+ Metrics that declare no category.
+
+
.. py:class:: InsertOp
Bases: :py:obj:`enum.Enum`
diff --git a/_sources/autoapi/datafusion/expr/index.rst.txt b/_sources/autoapi/datafusion/expr/index.rst.txt
index 3795cdf1..58fecc20 100644
--- a/_sources/autoapi/datafusion/expr/index.rst.txt
+++ b/_sources/autoapi/datafusion/expr/index.rst.txt
@@ -341,6 +341,13 @@ Module Contents
portable across minor versions. See :meth:`to_bytes` for
details on what travels by value vs. by reference.
+ The encoding side honors a driver-side sender context installed
+ via :func:`datafusion.ipc.set_sender_ctx` — that is how
+ :meth:`SessionContext.with_python_udf_inlining` propagates
+ through ``pickle.dumps``. The sender context is read by
+ ``__reduce__``, so :func:`copy.copy` and :func:`copy.deepcopy`
+ — which also go through ``__reduce__`` — pick it up too.
+
.. rubric:: Examples
>>> import pickle
@@ -349,13 +356,6 @@ Module Contents
>>> pickle.loads(pickle.dumps(e)).canonical_name()
'a * Int64(2)'
- The encoding side honors a driver-side sender context installed
- via :func:`datafusion.ipc.set_sender_ctx` — that is how
- :meth:`SessionContext.with_python_udf_inlining` propagates
- through ``pickle.dumps``. The sender context is read by
- ``__reduce__``, so :func:`copy.copy` and :func:`copy.deepcopy`
- — which also go through ``__reduce__`` — pick it up too.
-
.. py:method:: __repr__() -> str
@@ -631,8 +631,9 @@ Module Contents
Only evaluate distinct values for an aggregate function.
This function will create an :py:class:`ExprFuncBuilder` that can be used to
- set parameters for either window or aggregate functions. If used on any other
- type of expression, an error will be generated when ``build()`` is called.
+ set parameters for either window or aggregate functions. It raises on a window
+ function unless that is an aggregate used as one. If used on any other type of
+ expression, an error will be generated when ``build()`` is called.
@@ -671,8 +672,9 @@ Module Contents
Filter an aggregate function.
This function will create an :py:class:`ExprFuncBuilder` that can be used to
- set parameters for either window or aggregate functions. If used on any other
- type of expression, an error will be generated when ``build()`` is called.
+ set parameters for either window or aggregate functions. It raises on a window
+ function unless that is an aggregate used as one. If used on any other type of
+ expression, an error will be generated when ``build()`` is called.
@@ -705,6 +707,13 @@ Module Contents
(sufficient for built-ins and Python UDFs, plus any UDFs
registered on the global context).
+ Note that the global context carries no extension codecs, so
+ falling back to it cannot decode a payload written by one. A
+ worker that must decode extension-codec payloads has to install a
+ context carrying those codecs — see
+ :func:`datafusion.ipc.set_worker_ctx` and
+ :meth:`SessionContext.with_extensions`.
+
.. warning:: Security
Decoding may invoke ``cloudpickle.loads`` on bytes embedded
in the payload, which executes arbitrary Python code. Treat
@@ -744,6 +753,12 @@ Module Contents
+ .. py:method:: is_nan() -> Expr
+
+ Returns true if a given number is +NaN or -NaN otherwise returns false.
+
+
+
.. py:method:: is_not_null() -> Expr
Returns ``True`` if this expression is not null.
@@ -891,12 +906,19 @@ Module Contents
.. py:method:: over(window: Window) -> Expr
- Turn an aggregate function into a window function.
+ Evaluate an aggregate or window function over a window.
- This function turns any aggregate function into a window function. With the
+ On an aggregate function this turns it into a window function. With the
exception of ``partition_by``, how each of the parameters is used is determined
by the underlying aggregate function.
+ On an aggregate, the ``null_treatment``, ``filter``, and ``distinct`` options
+ it was built with are kept, and an ``order_by`` raises; see
+ :ref:`aggregate_over_options`.
+
+ On a window function that already has a ``partition_by``, ``order_by``, or
+ ``window_frame``, this raises; see :ref:`window_function_chaining`.
+
:param window: Window definition
@@ -906,8 +928,10 @@ Module Contents
Set the partitioning for a window function.
This function will create an :py:class:`ExprFuncBuilder` that can be used to
- set parameters for either window or aggregate functions. If used on any other
- type of expression, an error will be generated when ``build()`` is called.
+ set parameters for either window or aggregate functions. It raises on an
+ aggregate function that has not been turned into a window function with
+ :py:meth:`over`. If used on any other type of expression, an error will be
+ generated when ``build()`` is called.
@@ -1016,7 +1040,7 @@ Module Contents
- .. py:method:: sort(ascending: bool = True, nulls_first: bool = True) -> SortExpr
+ .. py:method:: sort(ascending: bool = True, nulls_first: bool = False) -> SortExpr
Creates a sort :py:class:`Expr` from an existing :py:class:`Expr`.
@@ -1201,11 +1225,13 @@ Module Contents
.. py:method:: window_frame(window_frame: WindowFrame) -> ExprFuncBuilder
- Set the frame fora window function.
+ Set the frame for a window function.
This function will create an :py:class:`ExprFuncBuilder` that can be used to
- set parameters for either window or aggregate functions. If used on any other
- type of expression, an error will be generated when ``build()`` is called.
+ set parameters for either window or aggregate functions. It raises on an
+ aggregate function that has not been turned into a window function with
+ :py:meth:`over`. If used on any other type of expression, an error will be
+ generated when ``build()`` is called.
diff --git a/_sources/autoapi/datafusion/extensions/index.rst.txt b/_sources/autoapi/datafusion/extensions/index.rst.txt
new file mode 100644
index 00000000..08745ee1
--- /dev/null
+++ b/_sources/autoapi/datafusion/extensions/index.rst.txt
@@ -0,0 +1,345 @@
+datafusion.extensions
+=====================
+
+.. py:module:: datafusion.extensions
+
+.. autoapi-nested-parse::
+
+ Protocols and value types for installing extensions on a session context.
+
+ An *extension* is a reusable configuration object — typically shipped by a
+ separate compiled library — that contributes components to a
+ :py:class:`~datafusion.context.SessionContext`. It implements
+ :py:class:`SessionComponentsExportable` by returning a
+ :py:class:`SessionExtensionComponents` describing what it contributes, and is
+ installed with :py:meth:`~datafusion.context.SessionContext.with_extensions`::
+
+ ctx = SessionContext().with_extensions(MyLibraryExtension())
+
+ Codecs and planners install in two phases: every
+ :py:class:`SessionComponentsExportable` runs first and its codecs are installed,
+ then every :py:class:`SessionPlannerExportable` runs in argument order. A bundle
+ implements either hook or both. Bundle order is significant for planners, which
+ nest, and irrelevant for codecs, which accumulate.
+
+ Of the five names here, only the two bundle hooks are ``@runtime_checkable``,
+ because :py:meth:`~datafusion.context.SessionContext.with_extensions`
+ dispatches on them from Python. :py:class:`QueryPlannerExportable` and
+ :py:class:`PhysicalOptimizerRuleExportable` are type hints only, matching the
+ other capsule-getter protocols in :py:mod:`datafusion.user_defined` and
+ :py:mod:`datafusion.catalog`.
+
+ See :ref:`extension_bundles` in the online documentation for why the phases are
+ split and for a worked implementation.
+
+
+
+Classes
+-------
+
+.. autoapisummary::
+
+ datafusion.extensions.PhysicalOptimizerRuleExportable
+ datafusion.extensions.QueryPlannerExportable
+ datafusion.extensions.SessionComponentsExportable
+ datafusion.extensions.SessionExtensionComponents
+ datafusion.extensions.SessionPlannerExportable
+
+
+Module Contents
+---------------
+
+.. py:class:: PhysicalOptimizerRuleExportable
+
+ Bases: :py:obj:`Protocol`
+
+
+ Type hint for object that has __datafusion_physical_optimizer_rule__ PyCapsule.
+
+ The method returns a PyCapsule wrapping an ``FFI_PhysicalOptimizerRule``,
+ typically produced by a separate compiled extension. It takes **no
+ argument**: a rule needs neither a codec nor a task-context provider, so
+ there is nothing session-scoped to hand it.
+
+ Rules accumulate rather than replace. Install one with
+ :py:meth:`~datafusion.context.SessionContext.add_physical_optimizer_rule`
+ — see :ref:`extension_other_hooks`.
+
+ .. rubric:: Examples
+
+ The getter is the whole protocol, and a capsule is what it must return
+ — anything else is refused where it is installed rather than at plan
+ time:
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> ctx.add_physical_optimizer_rule(object())
+ Traceback (most recent call last):
+ ...
+ RuntimeError: "Invalid datafusion_physical_optimizer_rule...
+
+ Real usage. Skipped here (needs a built extension library); parsed out
+ of this docstring and run for real by
+ ``test_physical_optimizer_rule_docstring_example_still_runs`` in
+ ``datafusion-ffi-example``.
+
+ >>> from datafusion_ffi_example import MyPhysicalOptimizerRule # doctest: +SKIP
+ >>> ctx.add_physical_optimizer_rule(MyPhysicalOptimizerRule()) # doctest: +SKIP
+
+
+ .. py:method:: __datafusion_physical_optimizer_rule__() -> object
+
+
+.. py:class:: QueryPlannerExportable
+
+ Bases: :py:obj:`Protocol`
+
+
+ Type hint for object that has a __datafusion_query_planner__ PyCapsule.
+
+ The method returns a PyCapsule wrapping an ``FFI_QueryPlanner``, typically
+ produced by a separate compiled extension. ``session`` is a handle on the
+ session the planner is being installed on; take the extension codecs from
+ it rather than building your own, and duck-type it — see
+ :py:class:`~datafusion.user_defined.LogicalExtensionCodecExportable` for
+ ``session``.
+
+ Unlike the two bundle hooks in this module, this protocol is a type hint
+ only: it is not ``@runtime_checkable``, so ``isinstance`` against it raises
+ ``TypeError``.
+
+ .. rubric:: Examples
+
+ A :py:class:`~datafusion.context.SessionContext` satisfies this
+ protocol, which is what lets a foreign planner wrap the one a session
+ already has:
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> type(ctx.__datafusion_query_planner__(ctx)).__name__
+ 'PyCapsule'
+
+ The protocol itself is not runtime-checkable:
+
+ >>> from datafusion.extensions import QueryPlannerExportable
+ >>> try:
+ ... isinstance(ctx, QueryPlannerExportable)
+ ... except TypeError as e:
+ ... print("runtime_checkable" in str(e))
+ True
+
+
+ .. py:method:: __datafusion_query_planner__(session: Any) -> object
+
+
+.. py:class:: SessionComponentsExportable
+
+ Bases: :py:obj:`Protocol`
+
+
+ Type hint for extension bundles installable via ``with_extensions``.
+
+ Runtime-checkable, so ``isinstance`` answers whether an object implements
+ the protocol. Only the presence of the method is checked, which is the same
+ question :py:meth:`~datafusion.context.SessionContext.with_extensions` asks
+ before calling it.
+
+ Implementations are reusable configuration objects: they must create fresh
+ components on every call using the context supplied by
+ :py:meth:`~datafusion.context.SessionContext.with_extensions`, and must not
+ retain that context or cache the components they bound to it, since the
+ next call may install onto a different session. They should also avoid
+ mutating the context they are handed — a registration made during binding
+ is not rolled back if a later extension fails. See
+ :ref:`extension_bundles_transaction`.
+
+ A bundle that also contributes a query planner implements
+ :py:class:`SessionPlannerExportable` alongside this protocol.
+
+ :param ctx: The session the components will run on. Take the task-context
+ provider off it. Do **not** read its codec chains expecting to find
+ this call's codecs, including your own: this hook runs before
+ anything is installed, so ``ctx`` still carries whatever chains the
+ receiver had. :py:class:`SessionPlannerExportable` is the hook that
+ sees the completed chains — see :ref:`extension_bundles_two_phases`.
+
+ :returns: The codecs this bundle contributes.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionExtensionComponents
+ >>> from datafusion.extensions import SessionComponentsExportable
+ >>> class MyLibraryExtension:
+ ... def __datafusion_session_components__(self, ctx):
+ ... return SessionExtensionComponents()
+ >>> isinstance(MyLibraryExtension(), SessionComponentsExportable)
+ True
+ >>> isinstance(object(), SessionComponentsExportable)
+ False
+
+
+ .. py:method:: __datafusion_session_components__(ctx: datafusion.context.SessionContext) -> SessionExtensionComponents
+
+
+.. py:class:: SessionExtensionComponents
+
+ Components an extension contributes to a session context.
+
+ Returned by :py:meth:`SessionComponentsExportable.__datafusion_session_components__`
+ and consumed by
+ :py:meth:`~datafusion.context.SessionContext.with_extensions`. Every
+ component must be created against the context passed to that method;
+ components bound to a different session hold a task-context provider for
+ that other session and cannot be rebound.
+
+ Construction is keyword-only, so later releases can add component kinds
+ without changing what an existing call means.
+
+ Query planners are not listed here. They install in a second phase so each
+ can wrap the one before it — see :py:class:`SessionPlannerExportable`.
+
+ .. rubric:: Examples
+
+ A bundle that contributes no codecs is valid — a planner-only library
+ returns this, or omits the hook entirely:
+
+ >>> from datafusion import SessionExtensionComponents
+ >>> components = SessionExtensionComponents()
+ >>> components.logical_extension_codecs
+ ()
+
+ A bundle that contributes one kind of component names it, leaving the
+ rest empty. Here the codec is a capsule wrapped in an object that
+ declares the id its payloads will carry:
+
+ >>> from datafusion import SessionContext
+ >>> class NamedCodec:
+ ... __datafusion_codec_id__ = "my_library.v1"
+ ...
+ ... def __init__(self, capsule):
+ ... self._capsule = capsule
+ ...
+ ... def __datafusion_logical_extension_codec__(self, session=None):
+ ... return self._capsule
+
+ >>> ctx = SessionContext()
+ >>> components = SessionExtensionComponents(
+ ... logical_extension_codecs=(NamedCodec(
+ ... ctx.__datafusion_logical_extension_codec__()
+ ... ),)
+ ... )
+ >>> components.logical_extension_codecs[0].__datafusion_codec_id__
+ 'my_library.v1'
+ >>> components.physical_extension_codecs
+ ()
+
+ A single codec is not an iterable of codecs, and forgetting the
+ trailing comma is the easy way to write one by accident:
+
+ >>> SessionExtensionComponents(logical_extension_codecs=NamedCodec(ctx))
+ Traceback (most recent call last):
+ ...
+ TypeError: logical_extension_codecs must be an iterable of codec objects...
+
+
+ .. py:method:: __post_init__() -> None
+
+ Normalize each codec field to a tuple, rejecting what cannot become one.
+
+
+
+ .. py:attribute:: logical_extension_codecs
+ :type: tuple[datafusion.user_defined.LogicalExtensionCodecExportable, Ellipsis]
+ :value: ()
+
+
+ Logical codecs to add to the session's codec chain, in declaration order.
+
+ Objects exposing ``__datafusion_logical_extension_codec__``, never bare
+ ``PyCapsule`` objects — a codec's id is read off the object it is handed
+ over as. Any iterable is accepted and stored as a tuple. See
+ :ref:`extension_bundles_codecs_are_objects`.
+
+
+ .. py:attribute:: physical_extension_codecs
+ :type: tuple[datafusion.user_defined.PhysicalExtensionCodecExportable, Ellipsis]
+ :value: ()
+
+
+ Physical codecs to add to the session's codec chain, in declaration order.
+
+ As :py:attr:`logical_extension_codecs`, for
+ ``__datafusion_physical_extension_codec__``.
+
+
+.. py:class:: SessionPlannerExportable
+
+ Bases: :py:obj:`Protocol`
+
+
+ Type hint for extension bundles that contribute a query planner.
+
+ A session holds exactly one query planner, so planners compose by nesting
+ rather than by chaining: each wraps the one before it and delegates to it
+ for the work it does not handle.
+ :py:meth:`~datafusion.context.SessionContext.with_extensions` runs this
+ hook once per bundle that implements it, **in argument order**, handing
+ each the planner built so far. Returning a planner that wraps ``fallback``
+ puts this bundle *outside* the previous one, so the last bundle listed ends
+ up outermost and is consulted first.
+
+ The hook runs after every codec from every bundle is installed, and ``ctx``
+ is the context carrying those final chains. That ordering is the point: a
+ planner captured here sees the complete codec set, so a nested planner is
+ not left encoding through a chain that a later bundle has grown. See
+ :ref:`extension_bundles_two_phases`.
+
+ **Return ``None`` to contribute no planner**, leaving ``fallback`` in
+ place. That is the no-op, and it is not the same as returning ``fallback``:
+ the capsule the first bundle receives wraps the session's planner for
+ export, so handing it back installs that planner as a foreign one and every
+ later plan crosses an FFI boundary that was not there before. A bundle that
+ decides at runtime it has nothing to contribute returns ``None``.
+
+ Ignoring ``fallback`` and returning a planner that does not delegate to it
+ is legal and means "replace" — but it discards every planner listed before
+ this one, including any the session already had.
+
+ :param ctx: The session the planner will run on, carrying the final codec
+ chains.
+ :param fallback: The planner built so far, as a ``PyCapsule``. For the first
+ bundle this is the session's existing planner, which is the
+ DataFusion default unless one was installed earlier.
+
+ :returns: A planner wrapping ``fallback``, or ``None`` to contribute none.
+
+ .. rubric:: Examples
+
+ A real library returns its own planner wrapping ``fallback``, e.g.
+ ``my_library.Planner(fallback=fallback)``. The two degenerate cases are
+ worth contrasting, because both plan queries successfully and only one
+ of them is the no-op:
+
+ >>> from datafusion import SessionContext
+ >>> from datafusion.extensions import SessionPlannerExportable
+ >>> class Contributes:
+ ... def __datafusion_session_planner__(self, ctx, fallback):
+ ... return None # the no-op: session keeps its own planner
+ >>> class Replaces:
+ ... def __datafusion_session_planner__(self, ctx, fallback):
+ ... return fallback # installs it as a *foreign* planner
+ >>> for bundle in (Contributes(), Replaces()):
+ ... ctx = SessionContext().with_extensions(bundle)
+ ... ctx.sql("SELECT 1 AS n").collect()[0].column(0).to_pylist()
+ [1]
+ [1]
+
+ >>> isinstance(Contributes(), SessionPlannerExportable)
+ True
+ >>> isinstance(object(), SessionPlannerExportable)
+ False
+
+
+ .. py:method:: __datafusion_session_planner__(ctx: datafusion.context.SessionContext, fallback: types.CapsuleType) -> QueryPlannerExportable | types.CapsuleType | None
+
+
diff --git a/_sources/autoapi/datafusion/functions/index.rst.txt b/_sources/autoapi/datafusion/functions/index.rst.txt
index 6f35c578..38d2c88e 100644
--- a/_sources/autoapi/datafusion/functions/index.rst.txt
+++ b/_sources/autoapi/datafusion/functions/index.rst.txt
@@ -56,15 +56,18 @@ Functions
datafusion.functions.acosh
datafusion.functions.alias
datafusion.functions.any_match
+ datafusion.functions.any_value
datafusion.functions.approx_distinct
datafusion.functions.approx_median
datafusion.functions.approx_percentile_cont
datafusion.functions.approx_percentile_cont_with_weight
datafusion.functions.array
+ datafusion.functions.array_add
datafusion.functions.array_agg
datafusion.functions.array_any_match
datafusion.functions.array_any_value
datafusion.functions.array_append
+ datafusion.functions.array_avg
datafusion.functions.array_cat
datafusion.functions.array_compact
datafusion.functions.array_concat
@@ -77,6 +80,7 @@ Functions
datafusion.functions.array_except
datafusion.functions.array_extract
datafusion.functions.array_filter
+ datafusion.functions.array_first
datafusion.functions.array_has
datafusion.functions.array_has_all
datafusion.functions.array_has_any
@@ -93,6 +97,7 @@ Functions
datafusion.functions.array_position
datafusion.functions.array_positions
datafusion.functions.array_prepend
+ datafusion.functions.array_product
datafusion.functions.array_push_back
datafusion.functions.array_push_front
datafusion.functions.array_remove
@@ -104,8 +109,11 @@ Functions
datafusion.functions.array_replace_n
datafusion.functions.array_resize
datafusion.functions.array_reverse
+ datafusion.functions.array_scale
datafusion.functions.array_slice
datafusion.functions.array_sort
+ datafusion.functions.array_subtract
+ datafusion.functions.array_sum
datafusion.functions.array_to_string
datafusion.functions.array_transform
datafusion.functions.array_union
@@ -175,6 +183,7 @@ Functions
datafusion.functions.exp
datafusion.functions.extract
datafusion.functions.factorial
+ datafusion.functions.file_row_index
datafusion.functions.find_in_set
datafusion.functions.first_value
datafusion.functions.flatten
@@ -190,7 +199,9 @@ Functions
datafusion.functions.in_list
datafusion.functions.initcap
datafusion.functions.inner_product
+ datafusion.functions.input_file_name
datafusion.functions.instr
+ datafusion.functions.is_nan
datafusion.functions.isnan
datafusion.functions.iszero
datafusion.functions.lag
@@ -203,9 +214,11 @@ Functions
datafusion.functions.left
datafusion.functions.length
datafusion.functions.levenshtein
+ datafusion.functions.list_add
datafusion.functions.list_any_match
datafusion.functions.list_any_value
datafusion.functions.list_append
+ datafusion.functions.list_avg
datafusion.functions.list_cat
datafusion.functions.list_compact
datafusion.functions.list_concat
@@ -218,6 +231,7 @@ Functions
datafusion.functions.list_except
datafusion.functions.list_extract
datafusion.functions.list_filter
+ datafusion.functions.list_first
datafusion.functions.list_has
datafusion.functions.list_has_all
datafusion.functions.list_has_any
@@ -235,6 +249,7 @@ Functions
datafusion.functions.list_position
datafusion.functions.list_positions
datafusion.functions.list_prepend
+ datafusion.functions.list_product
datafusion.functions.list_push_back
datafusion.functions.list_push_front
datafusion.functions.list_remove
@@ -246,8 +261,11 @@ Functions
datafusion.functions.list_replace_n
datafusion.functions.list_resize
datafusion.functions.list_reverse
+ datafusion.functions.list_scale
datafusion.functions.list_slice
datafusion.functions.list_sort
+ datafusion.functions.list_subtract
+ datafusion.functions.list_sum
datafusion.functions.list_to_string
datafusion.functions.list_transform
datafusion.functions.list_union
@@ -292,6 +310,7 @@ Functions
datafusion.functions.power
datafusion.functions.quantile_cont
datafusion.functions.radians
+ datafusion.functions.rand
datafusion.functions.random
datafusion.functions.range
datafusion.functions.rank
@@ -339,6 +358,7 @@ Functions
datafusion.functions.substr
datafusion.functions.substr_index
datafusion.functions.substring
+ datafusion.functions.substring_index
datafusion.functions.sum
datafusion.functions.tan
datafusion.functions.tanh
@@ -451,6 +471,40 @@ Package Contents
.. seealso:: This is an alias for :py:func:`array_any_match`.
+.. py:function:: any_value(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
+
+ Returns an arbitrary non-null value from each group.
+
+ Returns NULL if every value in the group is NULL. Which value is returned
+ is not specified and may differ between runs.
+
+ If using the builder functions described in :ref:`aggregation` this function ignores
+ the options ``order_by``, ``null_treatment``, and ``distinct``.
+
+ :param expression: Argument to pick a value from
+ :param filter: If provided, only consider rows for which the filter is True
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [None, 7, None]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.any_value(dfn.col("a")).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 7
+
+ >>> df = ctx.from_pydict({"a": [None, 7, 8], "b": [1, 2, 3]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.any_value(
+ ... dfn.col("a"),
+ ... filter=dfn.col("b") > dfn.lit(2)
+ ... ).alias("v")]
+ ... )
+ >>> result.collect_column("v")[0].as_py()
+ 8
+
+
.. py:function:: approx_distinct(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Returns the approximate number of distinct values.
@@ -459,7 +513,7 @@ Package Contents
will approximate the number of distinct entries. It may return significantly faster
than :py:func:`count` for some DataFrames.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param expression: Values to check for distinct entries
@@ -492,7 +546,7 @@ Package Contents
This aggregate function is similar to :py:func:`median`, but it will only
approximate the median. It may return significantly faster for some DataFrames.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by`` and ``null_treatment``, and ``distinct``.
:param expression: Values to find the median for
@@ -532,8 +586,10 @@ Package Contents
compute the percentile. You can limit the number of bins used in this algorithm by
setting the ``num_centroids`` parameter.
- If using the builder functions described in ref:`_aggregation` this function ignores
- the options ``order_by``, ``null_treatment``, and ``distinct``.
+ If using the builder functions described in :ref:`aggregation` this function ignores
+ the options ``null_treatment`` and ``distinct``. A chained ``order_by`` sets only
+ the sort direction; the percentile is still computed over ``sort_expression``,
+ so pass the same expression.
:param sort_expression: Values for which to find the approximate percentile
:param percentile: This must be between 0.0 and 1.0, inclusive
@@ -568,8 +624,10 @@ Package Contents
This aggregate function is similar to :py:func:`approx_percentile_cont` except that
it uses the associated associated weights.
- If using the builder functions described in ref:`_aggregation` this function ignores
- the options ``order_by``, ``null_treatment``, and ``distinct``.
+ If using the builder functions described in :ref:`aggregation` this function ignores
+ the option ``null_treatment`` and rejects ``distinct``. A chained ``order_by``
+ sets only the sort direction; the percentile is still computed over
+ ``sort_expression``, so pass the same expression.
:param sort_expression: Values for which to find the approximate percentile
:param weight: Relative weight for each of the values in ``expression``
@@ -605,6 +663,26 @@ Package Contents
.. seealso:: This is an alias for :py:func:`make_array`.
+.. py:function:: array_add(array1: datafusion.expr.Expr, array2: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the element-wise sum of two numeric arrays of equal length.
+
+ A NULL element in either input produces a NULL at that position. Execution
+ fails if the arrays in a row have different lengths.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict(
+ ... {"a": [[1.0, 2.0, 3.0]], "b": [[10.0, 20.0, 30.0]]}
+ ... )
+ >>> result = df.select(
+ ... dfn.functions.array_add(dfn.col("a"), dfn.col("b")).alias("result")
+ ... )
+ >>> result.collect_column("result")[0].as_py()
+ [11.0, 22.0, 33.0]
+
+
.. py:function:: array_agg(expression: datafusion.expr.Expr, distinct: bool = False, filter: datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None) -> datafusion.expr.Expr
Aggregate values into an array.
@@ -613,7 +691,7 @@ Package Contents
consider :py:func:`array_sort` after aggregation.
[Issue Tracker](https://github.com/apache/datafusion/issues/12371)
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the option ``null_treatment``.
:param expression: Values to combine into an array
@@ -708,6 +786,24 @@ Package Contents
[1, 2, 3, 4]
+.. py:function:: array_avg(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the arithmetic mean of the elements of a numeric array.
+
+ NULL elements are skipped and excluded from the count. Returns NULL if the
+ array is NULL, empty, or contains only NULL elements.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1.0, None, 3.0]]})
+ >>> result = df.select(
+ ... dfn.functions.array_avg(dfn.col("a")).alias("result")
+ ... )
+ >>> result.collect_column("result")[0].as_py()
+ 2.0
+
+
.. py:function:: array_cat(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
Concatenates the input arrays.
@@ -878,6 +974,36 @@ Package Contents
.. seealso:: :py:func:`array_transform`, :py:func:`array_any_match`, :py:func:`lambda_`.
+.. py:function:: array_first(array: datafusion.expr.Expr, predicate: datafusion.expr.Expr | collections.abc.Callable[Ellipsis, Any]) -> datafusion.expr.Expr
+
+ Return the first element of ``array`` for which ``predicate`` is ``True``.
+
+ ``predicate`` may be a Python callable, converted to a lambda
+ automatically, or an explicit lambda built with :py:func:`lambda_`. It must
+ return a boolean expression. Returns NULL if no element matches.
+
+ .. rubric:: Examples
+
+ Using a Python callable:
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1, 2, 3, 4]]})
+ >>> df.select(
+ ... F.array_first(col("a"), lambda v: v > 2).alias("f")
+ ... ).collect_column("f")[0].as_py()
+ 3
+
+ Using an explicit lambda built with :py:func:`lambda_`:
+
+ >>> predicate = F.lambda_(["v"], F.lambda_var("v") > lit(2))
+ >>> df.select(
+ ... F.array_first(col("a"), predicate).alias("f")
+ ... ).collect_column("f")[0].as_py()
+ 3
+
+ .. seealso:: :py:func:`array_filter`, :py:func:`array_any_match`, :py:func:`lambda_`.
+
+
.. py:function:: array_has(first_array: datafusion.expr.Expr, second_array: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns true if the element appears in the first array, otherwise false.
@@ -952,7 +1078,7 @@ Package Contents
[2, 3]
-.. py:function:: array_join(expr: datafusion.expr.Expr, delimiter: datafusion.expr.Expr | str) -> datafusion.expr.Expr
+.. py:function:: array_join(expr: datafusion.expr.Expr, delimiter: datafusion.expr.Expr | str, null_string: datafusion.expr.Expr | str | None = None) -> datafusion.expr.Expr
Converts each element to its text representation.
@@ -1132,6 +1258,24 @@ Package Contents
[0, 1, 2]
+.. py:function:: array_product(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the product of the elements of a numeric array.
+
+ NULL elements are skipped. Returns NULL if the array is NULL, empty, or
+ contains only NULL elements.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[2.0, None, 3.0]]})
+ >>> result = df.select(
+ ... dfn.functions.array_product(dfn.col("a")).alias("result")
+ ... )
+ >>> result.collect_column("result")[0].as_py()
+ 6.0
+
+
.. py:function:: array_push_back(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
Appends an element to the end of an array.
@@ -1286,6 +1430,24 @@ Package Contents
[3, 2, 1]
+.. py:function:: array_scale(array: datafusion.expr.Expr, scalar: datafusion.expr.Expr | float) -> datafusion.expr.Expr
+
+ Multiplies each element of a numeric array by ``scalar``.
+
+ A NULL element produces a NULL at that position. Returns NULL if ``scalar``
+ is NULL.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1.0, 2.0, 3.0]]})
+ >>> result = df.select(
+ ... dfn.functions.array_scale(dfn.col("a"), 2.0).alias("result")
+ ... )
+ >>> result.collect_column("result")[0].as_py()
+ [2.0, 4.0, 6.0]
+
+
.. py:function:: array_slice(array: datafusion.expr.Expr, begin: datafusion.expr.Expr | int, end: datafusion.expr.Expr | int, stride: datafusion.expr.Expr | int | None = None) -> datafusion.expr.Expr
Returns a slice of the array.
@@ -1337,18 +1499,69 @@ Package Contents
[None, 3, 1]
-.. py:function:: array_to_string(expr: datafusion.expr.Expr, delimiter: datafusion.expr.Expr | str) -> datafusion.expr.Expr
+.. py:function:: array_subtract(array1: datafusion.expr.Expr, array2: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the element-wise difference of two numeric arrays of equal length.
+
+ Computes ``array1[i] - array2[i]``. A NULL element in either input produces
+ a NULL at that position. Execution fails if the arrays in a row have
+ different lengths.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict(
+ ... {"a": [[10.0, 20.0, 30.0]], "b": [[1.0, 2.0, 3.0]]}
+ ... )
+ >>> result = df.select(
+ ... dfn.functions.array_subtract(
+ ... dfn.col("a"), dfn.col("b")
+ ... ).alias("result")
+ ... )
+ >>> result.collect_column("result")[0].as_py()
+ [9.0, 18.0, 27.0]
+
+
+.. py:function:: array_sum(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the sum of the elements of a numeric array.
+
+ NULL elements are skipped. Returns NULL if the array is NULL, empty, or
+ contains only NULL elements.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [[1.0, None, 3.0]]})
+ >>> result = df.select(
+ ... dfn.functions.array_sum(dfn.col("a")).alias("result")
+ ... )
+ >>> result.collect_column("result")[0].as_py()
+ 4.0
+
+
+.. py:function:: array_to_string(expr: datafusion.expr.Expr, delimiter: datafusion.expr.Expr | str, null_string: datafusion.expr.Expr | str | None = None) -> datafusion.expr.Expr
Converts each element to its text representation.
+ NULL elements are omitted unless ``null_string`` is given, in which case it
+ is written in their place.
+
.. rubric:: Examples
>>> ctx = dfn.SessionContext()
- >>> df = ctx.from_pydict({"a": [[1, 2, 3]]})
+ >>> df = ctx.from_pydict({"a": [[1, None, 3]]})
>>> result = df.select(
... dfn.functions.array_to_string(dfn.col("a"), ",").alias("s"))
>>> result.collect_column("s")[0].as_py()
- '1,2,3'
+ '1,3'
+
+ >>> result = df.select(
+ ... dfn.functions.array_to_string(
+ ... dfn.col("a"), ",", null_string="*"
+ ... ).alias("s"))
+ >>> result.collect_column("s")[0].as_py()
+ '1,*,3'
.. py:function:: array_transform(array: datafusion.expr.Expr, transform: datafusion.expr.Expr | collections.abc.Callable[Ellipsis, Any]) -> datafusion.expr.Expr
@@ -1631,7 +1844,7 @@ Package Contents
This aggregate function expects a numeric expression and will return a float.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by`` and ``null_treatment``.
:param expression: Values to combine into an array
@@ -1666,16 +1879,17 @@ Package Contents
2.0
-.. py:function:: bit_and(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
+.. py:function:: bit_and(expression: datafusion.expr.Expr, distinct: bool = False, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Computes the bitwise AND of the argument.
This aggregate function will bitwise compare every value in the input partition.
- If using the builder functions described in ref:`_aggregation` this function ignores
- the options ``order_by``, ``null_treatment``, and ``distinct``.
+ If using the builder functions described in :ref:`aggregation` this function ignores
+ the options ``order_by`` and ``null_treatment``.
:param expression: Argument to perform bitwise calculation on
+ :param distinct: If True, evaluate each unique value of expression only once
:param filter: If provided, only compute against rows for which the filter is True
.. rubric:: Examples
@@ -1712,16 +1926,17 @@ Package Contents
8
-.. py:function:: bit_or(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
+.. py:function:: bit_or(expression: datafusion.expr.Expr, distinct: bool = False, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Computes the bitwise OR of the argument.
This aggregate function will bitwise compare every value in the input partition.
- If using the builder functions described in ref:`_aggregation` this function ignores
- the options ``order_by``, ``null_treatment``, and ``distinct``.
+ If using the builder functions described in :ref:`aggregation` this function ignores
+ the options ``order_by`` and ``null_treatment``.
:param expression: Argument to perform bitwise calculation on
+ :param distinct: If True, evaluate each unique value of expression only once
:param filter: If provided, only compute against rows for which the filter is True
.. rubric:: Examples
@@ -1753,7 +1968,7 @@ Package Contents
This aggregate function will bitwise compare every value in the input partition.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by`` and ``null_treatment``.
:param expression: Argument to perform bitwise calculation on
@@ -1790,7 +2005,7 @@ Package Contents
This aggregate function will compare every value in the input partition. These are
expected to be boolean values.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param expression: Argument to perform calculation on
@@ -1827,7 +2042,7 @@ Package Contents
This aggregate function will compare every value in the input partition. These are
expected to be boolean values.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param expression: Argument to perform calculation on
@@ -1857,9 +2072,9 @@ Package Contents
False
-.. py:function:: btrim(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
+.. py:function:: btrim(arg: datafusion.expr.Expr, characters: datafusion.expr.Expr | str | None = None) -> datafusion.expr.Expr
- Removes all characters, spaces by default, from both sides of a string.
+ Removes ``characters``, spaces by default, from both sides of a string.
.. rubric:: Examples
@@ -1869,6 +2084,15 @@ Package Contents
>>> trim_df.collect_column("trimmed")[0].as_py()
'a'
+ Trim a different set of characters:
+
+ >>> df = ctx.from_pydict({"a": ["xxaxx"]})
+ >>> trim_df = df.select(
+ ... dfn.functions.btrim(dfn.col("a"), characters="x").alias("trimmed")
+ ... )
+ >>> trim_df.collect_column("trimmed")[0].as_py()
+ 'a'
+
.. py:function:: cardinality(array: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -2075,7 +2299,7 @@ Package Contents
This aggregate function expects both values to be numeric and will return a float.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param value_y: The dependent variable for correlation
@@ -2204,7 +2428,7 @@ Package Contents
This aggregate function will count the non-null rows provided in the expression.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by`` and ``null_treatment``.
:param expressions: Argument to perform bitwise calculation on
@@ -2238,8 +2462,9 @@ Package Contents
This aggregate function will count all of the rows in the partition.
- If using the builder functions described in ref:`_aggregation` this function ignores
- the options ``order_by``, ``distinct``, and ``null_treatment``.
+ If using the builder functions described in :ref:`aggregation` this function ignores
+ the options ``order_by`` and ``null_treatment``. ``distinct`` counts the distinct
+ values of the constant ``1``, so the result is 1 for any non-empty input.
:param filter: If provided, only count rows for which the filter is True
@@ -2274,7 +2499,7 @@ Package Contents
This aggregate function expects both values to be numeric and will return a float.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param value_y: The dependent variable for covariance
@@ -2313,7 +2538,7 @@ Package Contents
This aggregate function expects both values to be numeric and will return a float.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param value_y: The dependent variable for covariance
@@ -2693,6 +2918,30 @@ Package Contents
6
+.. py:function:: file_row_index() -> datafusion.expr.Expr
+
+ Returns the zero-based position of the current row within its source file.
+
+ The index restarts at zero for each file, so rows from different files in one
+ scan can share a value. Only valid inside a scan of a Parquet table;
+ evaluating it anywhere else raises an error.
+
+ .. rubric:: Examples
+
+ >>> import tempfile, os
+ >>> import pyarrow as pa, pyarrow.parquet as pq
+ >>> tmp = tempfile.mkdtemp()
+ >>> path = os.path.join(tmp, "data.parquet")
+ >>> pq.write_table(pa.table({"a": [10, 20, 30]}), path)
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.read_parquet(path).filter(dfn.col("a") > dfn.lit(10))
+ >>> result = df.select(dfn.functions.file_row_index().alias("i"))
+ >>> result.collect_column("i").to_pylist()
+ [1, 2]
+
+ .. seealso:: :py:func:`input_file_name`.
+
+
.. py:function:: find_in_set(string: datafusion.expr.Expr, string_list: datafusion.expr.Expr | str) -> datafusion.expr.Expr
Find a string in a list of strings.
@@ -2718,7 +2967,7 @@ Package Contents
This aggregate function will return the first value in the partition.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the option ``distinct``.
:param expression: Argument to perform bitwise calculation on
@@ -2810,38 +3059,40 @@ Package Contents
4
-.. py:function:: gen_series(start: datafusion.expr.Expr, stop: datafusion.expr.Expr, step: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
+.. py:function:: gen_series(stop: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+ gen_series(start: datafusion.expr.Expr | int, stop: datafusion.expr.Expr | int, step: datafusion.expr.Expr | int | None = None) -> datafusion.expr.Expr
- Creates a list of values in the range between start and stop.
+ Creates a list of values from ``start`` up to and including ``stop``.
- Unlike :py:func:`range`, this includes the upper bound.
+ Unlike :py:func:`range`, this includes the upper bound. With a single
+ argument it is ``stop`` and the series starts at 0.
.. rubric:: Examples
>>> ctx = dfn.SessionContext()
>>> df = ctx.from_pydict({"a": [0]})
- >>> result = df.select(
- ... dfn.functions.gen_series(
- ... dfn.lit(1), dfn.lit(5),
- ... ).alias("result"))
+ >>> result = df.select(dfn.functions.gen_series(3).alias("result"))
+ >>> result.collect_column("result")[0].as_py()
+ [0, 1, 2, 3]
+
+ Specify a ``stop``:
+
+ >>> result = df.select(dfn.functions.gen_series(1, stop=5).alias("result"))
>>> result.collect_column("result")[0].as_py()
[1, 2, 3, 4, 5]
- Specify a custom ``step``:
+ Specify a ``step``:
>>> result = df.select(
- ... dfn.functions.gen_series(
- ... dfn.lit(1), dfn.lit(10), step=dfn.lit(3),
- ... ).alias("result"))
+ ... dfn.functions.gen_series(1, stop=10, step=3).alias("result"))
>>> result.collect_column("result")[0].as_py()
[1, 4, 7, 10]
-.. py:function:: generate_series(start: datafusion.expr.Expr, stop: datafusion.expr.Expr, step: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
-
- Creates a list of values in the range between start and stop.
+.. py:function:: generate_series(stop: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+ generate_series(start: datafusion.expr.Expr | int, stop: datafusion.expr.Expr | int, step: datafusion.expr.Expr | int | None = None) -> datafusion.expr.Expr
- Unlike :py:func:`range`, this includes the upper bound.
+ Creates a list of values from ``start`` up to and including ``stop``.
.. seealso:: This is an alias for :py:func:`gen_series`.
@@ -3063,6 +3314,29 @@ Package Contents
True
+.. py:function:: input_file_name() -> datafusion.expr.Expr
+
+ Returns the path of the file that produced the current row.
+
+ Only valid inside a scan of a file-backed table; evaluating it anywhere
+ else raises an error.
+
+ .. rubric:: Examples
+
+ >>> import tempfile, os
+ >>> import pyarrow as pa, pyarrow.parquet as pq
+ >>> tmp = tempfile.mkdtemp()
+ >>> path = os.path.join(tmp, "data.parquet")
+ >>> pq.write_table(pa.table({"a": [1, 2]}), path)
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.read_parquet(path)
+ >>> result = df.select(dfn.functions.input_file_name().alias("f"))
+ >>> result.collect_column("f")[0].as_py().endswith("data.parquet")
+ True
+
+ .. seealso:: :py:func:`file_row_index`.
+
+
.. py:function:: instr(string: datafusion.expr.Expr, substring: datafusion.expr.Expr | str) -> datafusion.expr.Expr
Finds the position from where the ``substring`` matches the ``string``.
@@ -3070,6 +3344,11 @@ Package Contents
.. seealso:: This is an alias for :py:func:`strpos`.
+.. py:function:: is_nan(expr: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Alias for :func:`isnan`.
+
+
.. py:function:: isnan(expr: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns true if a given number is +NaN or -NaN otherwise returns false.
@@ -3096,7 +3375,7 @@ Package Contents
True
-.. py:function:: lag(arg: datafusion.expr.Expr, shift_offset: int = 1, default_value: Any | None = None, partition_by: list[datafusion.expr.Expr] | datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None) -> datafusion.expr.Expr
+.. py:function:: lag(arg: datafusion.expr.Expr, shift_offset: int = 1, default_value: Any | None = None, partition_by: list[datafusion.expr.Expr] | datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None, null_treatment: datafusion.common.NullTreatment | None = None) -> datafusion.expr.Expr
Create a lag window function.
@@ -3123,6 +3402,8 @@ Package Contents
:param partition_by: Expressions to partition the window frame on.
:param order_by: Set ordering within the window frame. Accepts
column names or expressions.
+ :param null_treatment: Set to ``IGNORE_NULLS`` to skip null values when
+ counting ``shift_offset`` rows.
.. rubric:: Examples
@@ -3147,6 +3428,16 @@ Package Contents
>>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("lag").to_pylist()
[0, 1, 0]
+ >>> df = ctx.from_pydict({"i": [1, 2, 3, 4], "v": [1, None, None, 4]})
+ >>> result = df.select(
+ ... dfn.col("i"),
+ ... dfn.functions.lag(
+ ... dfn.col("v"), order_by="i",
+ ... null_treatment=dfn.common.NullTreatment.IGNORE_NULLS,
+ ... ).alias("lag"))
+ >>> result.sort(dfn.col("i")).collect_column("lag").to_pylist()
+ [None, 1, 1, 1]
+
.. py:function:: lambda_(params: list[str], body: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -3202,7 +3493,7 @@ Package Contents
This aggregate function will return the last value in the partition.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the option ``distinct``.
:param expression: Argument to perform bitwise calculation on
@@ -3251,7 +3542,7 @@ Package Contents
12
-.. py:function:: lead(arg: datafusion.expr.Expr, shift_offset: int = 1, default_value: Any | None = None, partition_by: list[datafusion.expr.Expr] | datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None) -> datafusion.expr.Expr
+.. py:function:: lead(arg: datafusion.expr.Expr, shift_offset: int = 1, default_value: Any | None = None, partition_by: list[datafusion.expr.Expr] | datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None, null_treatment: datafusion.common.NullTreatment | None = None) -> datafusion.expr.Expr
Create a lead window function.
@@ -3273,7 +3564,7 @@ Package Contents
+--------+------+-----+
To set window function parameters use the window builder approach described in the
- ref:`_window_functions` online documentation.
+ :ref:`window_functions` online documentation.
:param arg: Value to return
:param shift_offset: Number of rows following the current row.
@@ -3281,6 +3572,8 @@ Package Contents
:param partition_by: Expressions to partition the window frame on.
:param order_by: Set ordering within the window frame. Accepts
column names or expressions.
+ :param null_treatment: Set to ``IGNORE_NULLS`` to skip null values when
+ counting ``shift_offset`` rows.
.. rubric:: Examples
@@ -3305,6 +3598,16 @@ Package Contents
>>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("lead").to_pylist()
[2, 0, 0]
+ >>> df = ctx.from_pydict({"i": [1, 2, 3, 4], "v": [1, None, None, 4]})
+ >>> result = df.select(
+ ... dfn.col("i"),
+ ... dfn.functions.lead(
+ ... dfn.col("v"), order_by="i",
+ ... null_treatment=dfn.common.NullTreatment.IGNORE_NULLS,
+ ... ).alias("lead"))
+ >>> result.sort(dfn.col("i")).collect_column("lead").to_pylist()
+ [4, 4, 4, None]
+
.. py:function:: least(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -3365,6 +3668,13 @@ Package Contents
3
+.. py:function:: list_add(array1: datafusion.expr.Expr, array2: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the element-wise sum of two numeric lists of equal length.
+
+ .. seealso:: This is an alias for :py:func:`array_add`.
+
+
.. py:function:: list_any_match(array: datafusion.expr.Expr, predicate: datafusion.expr.Expr | collections.abc.Callable[Ellipsis, Any]) -> datafusion.expr.Expr
Return ``True`` if any element of a list satisfies a predicate.
@@ -3386,6 +3696,13 @@ Package Contents
.. seealso:: This is an alias for :py:func:`array_append`.
+.. py:function:: list_avg(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the arithmetic mean of the elements of a numeric list.
+
+ .. seealso:: This is an alias for :py:func:`array_avg`.
+
+
.. py:function:: list_cat(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
Concatenates the input arrays.
@@ -3470,6 +3787,13 @@ Package Contents
.. seealso:: This is an alias for :py:func:`array_filter`.
+.. py:function:: list_first(array: datafusion.expr.Expr, predicate: datafusion.expr.Expr | collections.abc.Callable[Ellipsis, Any]) -> datafusion.expr.Expr
+
+ Return the first element of a list for which a predicate is ``True``.
+
+ .. seealso:: This is an alias for :py:func:`array_first`.
+
+
.. py:function:: list_has(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
Returns true if the element appears in the array, otherwise false.
@@ -3505,7 +3829,7 @@ Package Contents
.. seealso:: This is an alias for :py:func:`array_intersect`.
-.. py:function:: list_join(expr: datafusion.expr.Expr, delimiter: datafusion.expr.Expr | str) -> datafusion.expr.Expr
+.. py:function:: list_join(expr: datafusion.expr.Expr, delimiter: datafusion.expr.Expr | str, null_string: datafusion.expr.Expr | str | None = None) -> datafusion.expr.Expr
Converts each element to its text representation.
@@ -3589,6 +3913,13 @@ Package Contents
.. seealso:: This is an alias for :py:func:`array_prepend`.
+.. py:function:: list_product(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the product of the elements of a numeric list.
+
+ .. seealso:: This is an alias for :py:func:`array_product`.
+
+
.. py:function:: list_push_back(array: datafusion.expr.Expr, element: datafusion.expr.Expr) -> datafusion.expr.Expr
Appends an element to the end of an array.
@@ -3672,6 +4003,13 @@ Package Contents
.. seealso:: This is an alias for :py:func:`array_reverse`.
+.. py:function:: list_scale(array: datafusion.expr.Expr, scalar: datafusion.expr.Expr | float) -> datafusion.expr.Expr
+
+ Multiplies each element of a numeric list by a scalar.
+
+ .. seealso:: This is an alias for :py:func:`array_scale`.
+
+
.. py:function:: list_slice(array: datafusion.expr.Expr, begin: datafusion.expr.Expr | int, end: datafusion.expr.Expr | int, stride: datafusion.expr.Expr | int | None = None) -> datafusion.expr.Expr
Returns a slice of the array.
@@ -3686,7 +4024,21 @@ Package Contents
.. seealso:: This is an alias for :py:func:`array_sort`.
-.. py:function:: list_to_string(expr: datafusion.expr.Expr, delimiter: datafusion.expr.Expr | str) -> datafusion.expr.Expr
+.. py:function:: list_subtract(array1: datafusion.expr.Expr, array2: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the element-wise difference of two numeric lists of equal length.
+
+ .. seealso:: This is an alias for :py:func:`array_subtract`.
+
+
+.. py:function:: list_sum(array: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Returns the sum of the elements of a numeric list.
+
+ .. seealso:: This is an alias for :py:func:`array_sum`.
+
+
+.. py:function:: list_to_string(expr: datafusion.expr.Expr, delimiter: datafusion.expr.Expr | str, null_string: datafusion.expr.Expr | str | None = None) -> datafusion.expr.Expr
Converts each element to its text representation.
@@ -3810,9 +4162,9 @@ Package Contents
'...the cat'
-.. py:function:: ltrim(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
+.. py:function:: ltrim(arg: datafusion.expr.Expr, characters: datafusion.expr.Expr | str | None = None) -> datafusion.expr.Expr
- Removes all characters, spaces by default, from the beginning of a string.
+ Removes ``characters``, spaces by default, from the beginning of a string.
.. rubric:: Examples
@@ -3822,6 +4174,15 @@ Package Contents
>>> trim_df.collect_column("trimmed")[0].as_py()
'a '
+ Trim a different set of characters:
+
+ >>> df = ctx.from_pydict({"a": ["xxaxx"]})
+ >>> trim_df = df.select(
+ ... dfn.functions.ltrim(dfn.col("a"), characters="x").alias("trimmed")
+ ... )
+ >>> trim_df.collect_column("trimmed")[0].as_py()
+ 'axx'
+
.. py:function:: make_array(*args: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -4010,7 +4371,7 @@ Package Contents
Aggregate function that returns the maximum value of the argument.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param expression: The value to find the maximum of
@@ -4049,7 +4410,7 @@ Package Contents
'5d41402abc4b2a76b9719d911017c592'
-.. py:function:: mean(expression: datafusion.expr.Expr, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
+.. py:function:: mean(expression: datafusion.expr.Expr, distinct: bool = False, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Returns the average (mean) value of the argument.
@@ -4063,7 +4424,7 @@ Package Contents
This aggregate function returns the median value of the expression for the given
aggregate function.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by`` and ``null_treatment``.
:param expression: The value to compute the median of
@@ -4095,7 +4456,7 @@ Package Contents
Aggregate function that returns the minimum value of the argument.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param expression: The value to find the minimum of
@@ -4184,7 +4545,7 @@ Package Contents
This aggregate function will return the n-th value in the partition.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the option ``distinct``.
:param expression: Argument to perform bitwise calculation on
@@ -4337,7 +4698,7 @@ Package Contents
5
-.. py:function:: order_by(expr: datafusion.expr.Expr, ascending: bool = True, nulls_first: bool = True) -> datafusion.expr.SortExpr
+.. py:function:: order_by(expr: datafusion.expr.Expr, ascending: bool = True, nulls_first: bool = False) -> datafusion.expr.SortExpr
Creates a new sort expression.
@@ -4416,18 +4777,21 @@ Package Contents
[0.0, 0.5, 1.0, 0.0, 1.0]
-.. py:function:: percentile_cont(sort_expression: datafusion.expr.Expr | datafusion.expr.SortExpr, percentile: float, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
+.. py:function:: percentile_cont(sort_expression: datafusion.expr.Expr | datafusion.expr.SortExpr, percentile: float, distinct: bool = False, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Computes the exact percentile of input values using continuous interpolation.
Unlike :py:func:`approx_percentile_cont`, this function computes the exact
percentile value rather than an approximation.
- If using the builder functions described in ref:`_aggregation` this function ignores
- the options ``order_by``, ``null_treatment``, and ``distinct``.
+ If using the builder functions described in :ref:`aggregation` this function ignores
+ the option ``null_treatment``. A chained ``order_by`` sets only the sort
+ direction; the percentile is still computed over ``sort_expression``, so pass
+ the same expression.
:param sort_expression: Values for which to find the percentile
:param percentile: This must be between 0.0 and 1.0, inclusive
+ :param distinct: If True, duplicate values are removed before computing
:param filter: If provided, only compute against rows for which the filter is True
.. rubric:: Examples
@@ -4449,6 +4813,14 @@ Package Contents
>>> result.collect_column("v")[0].as_py()
3.5
+ >>> df = ctx.from_pydict({"a": [1.0, 1.0, 1.0, 4.0]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.percentile_cont(
+ ... dfn.col("a"), 0.5, distinct=True,
+ ... ).alias("v")])
+ >>> result.collect_column("v")[0].as_py()
+ 2.5
+
.. py:function:: pi() -> datafusion.expr.Expr
@@ -4495,7 +4867,7 @@ Package Contents
8.0
-.. py:function:: quantile_cont(sort_expression: datafusion.expr.Expr | datafusion.expr.SortExpr, percentile: float, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
+.. py:function:: quantile_cont(sort_expression: datafusion.expr.Expr | datafusion.expr.SortExpr, percentile: float, distinct: bool = False, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Computes the exact percentile of input values using continuous interpolation.
@@ -4518,6 +4890,13 @@ Package Contents
True
+.. py:function:: rand() -> datafusion.expr.Expr
+
+ Returns a random value in the range ``0.0 <= x < 1.0``.
+
+ .. seealso:: This is an alias for :py:func:`random`.
+
+
.. py:function:: random() -> datafusion.expr.Expr
Returns a random value in the range ``0.0 <= x < 1.0``.
@@ -4534,16 +4913,31 @@ Package Contents
True
-.. py:function:: range(start: datafusion.expr.Expr, stop: datafusion.expr.Expr, step: datafusion.expr.Expr) -> datafusion.expr.Expr
+.. py:function:: range(stop: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+ range(start: datafusion.expr.Expr | int, stop: datafusion.expr.Expr | int, step: datafusion.expr.Expr | int | None = None) -> datafusion.expr.Expr
+
+ Create a list of values from ``start`` up to, but excluding, ``stop``.
- Create a list of values in the range between start and stop.
+ With a single argument it is ``stop`` and the range starts at 0, like
+ Python's built-in :py:class:`range`.
.. rubric:: Examples
>>> ctx = dfn.SessionContext()
>>> df = ctx.from_pydict({"a": [1]})
- >>> result = df.select(
- ... dfn.functions.range(dfn.lit(0), dfn.lit(5), dfn.lit(2)).alias("r"))
+ >>> result = df.select(dfn.functions.range(5).alias("r"))
+ >>> result.collect_column("r")[0].as_py()
+ [0, 1, 2, 3, 4]
+
+ Specify a ``stop``:
+
+ >>> result = df.select(dfn.functions.range(1, stop=5).alias("r"))
+ >>> result.collect_column("r")[0].as_py()
+ [1, 2, 3, 4]
+
+ Specify a ``step``:
+
+ >>> result = df.select(dfn.functions.range(0, stop=5, step=2).alias("r"))
>>> result.collect_column("r")[0].as_py()
[0, 2, 4]
@@ -4767,7 +5161,7 @@ Package Contents
This is a linear regression aggregate function. Only non-null pairs of the inputs
are evaluated.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param y: The linear regression dependent variable
@@ -4801,7 +5195,7 @@ Package Contents
This is a linear regression aggregate function. Only non-null pairs of the inputs
are evaluated.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param y: The linear regression dependent variable
@@ -4835,7 +5229,7 @@ Package Contents
This is a linear regression aggregate function. Only non-null pairs of the inputs
are evaluated.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param y: The linear regression dependent variable
@@ -4869,7 +5263,7 @@ Package Contents
This is a linear regression aggregate function. Only non-null pairs of the inputs
are evaluated.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param y: The linear regression dependent variable
@@ -4905,7 +5299,7 @@ Package Contents
This is a linear regression aggregate function. Only non-null pairs of the inputs
are evaluated.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param y: The linear regression dependent variable
@@ -4939,7 +5333,7 @@ Package Contents
This is a linear regression aggregate function. Only non-null pairs of the inputs
are evaluated.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param y: The linear regression dependent variable
@@ -4973,7 +5367,7 @@ Package Contents
This is a linear regression aggregate function. Only non-null pairs of the inputs
are evaluated.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param y: The linear regression dependent variable
@@ -5007,7 +5401,7 @@ Package Contents
This is a linear regression aggregate function. Only non-null pairs of the inputs
are evaluated.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param y: The linear regression dependent variable
@@ -5041,7 +5435,7 @@ Package Contents
This is a linear regression aggregate function. Only non-null pairs of the inputs
are evaluated.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param y: The linear regression dependent variable
@@ -5207,9 +5601,9 @@ Package Contents
'hi!!!'
-.. py:function:: rtrim(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
+.. py:function:: rtrim(arg: datafusion.expr.Expr, characters: datafusion.expr.Expr | str | None = None) -> datafusion.expr.Expr
- Removes all characters, spaces by default, from the end of a string.
+ Removes ``characters``, spaces by default, from the end of a string.
.. rubric:: Examples
@@ -5219,6 +5613,15 @@ Package Contents
>>> trim_df.collect_column("trimmed")[0].as_py()
' a'
+ Trim a different set of characters:
+
+ >>> df = ctx.from_pydict({"a": ["xxaxx"]})
+ >>> trim_df = df.select(
+ ... dfn.functions.rtrim(dfn.col("a"), characters="x").alias("trimmed")
+ ... )
+ >>> trim_df.collect_column("trimmed")[0].as_py()
+ 'xxa'
+
.. py:function:: sha224(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
@@ -5367,7 +5770,7 @@ Package Contents
Computes the standard deviation of the argument.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param expression: The value to find the minimum of
@@ -5397,7 +5800,7 @@ Package Contents
Computes the population standard deviation of the argument.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param expression: The value to find the minimum of
@@ -5433,7 +5836,7 @@ Package Contents
.. seealso:: This is an alias for :py:func:`stddev`.
-.. py:function:: string_agg(expression: datafusion.expr.Expr, delimiter: str, filter: datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None) -> datafusion.expr.Expr
+.. py:function:: string_agg(expression: datafusion.expr.Expr, delimiter: str, distinct: bool = False, filter: datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None) -> datafusion.expr.Expr
Concatenates the input strings.
@@ -5441,11 +5844,12 @@ Package Contents
separating them with the specified delimiter. Non-string values will be converted to
their string equivalents.
- If using the builder functions described in ref:`_aggregation` this function ignores
- the options ``distinct`` and ``null_treatment``.
+ If using the builder functions described in :ref:`aggregation` this function ignores
+ the option ``null_treatment``.
:param expression: Argument to perform bitwise calculation on
:param delimiter: Text to place between each value of expression
+ :param distinct: If True, each unique value of expression is included only once
:param filter: If provided, only compute against rows for which the filter is True
:param order_by: Set the ordering of the expression to evaluate. Accepts
column names or expressions.
@@ -5470,6 +5874,14 @@ Package Contents
>>> result.collect_column("s")[0].as_py()
'y,z'
+ >>> df = ctx.from_pydict({"a": ["y", "x", "y"]})
+ >>> result = df.aggregate(
+ ... [], [dfn.functions.string_agg(
+ ... dfn.col("a"), ",", distinct=True, order_by="a",
+ ... ).alias("s")])
+ >>> result.collect_column("s")[0].as_py()
+ 'x,y'
+
.. py:function:: string_to_array(string: datafusion.expr.Expr, delimiter: datafusion.expr.Expr | str, null_string: datafusion.expr.Expr | str | None = None) -> datafusion.expr.Expr
@@ -5538,9 +5950,9 @@ Package Contents
True
-.. py:function:: substr(string: datafusion.expr.Expr, position: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+.. py:function:: substr(string: datafusion.expr.Expr, position: datafusion.expr.Expr | int, length: datafusion.expr.Expr | int | None = None) -> datafusion.expr.Expr
- Substring from the ``position`` to the end.
+ Substring from the ``position``, to the end or for ``length`` characters.
.. rubric:: Examples
@@ -5551,6 +5963,13 @@ Package Contents
>>> result.collect_column("s")[0].as_py()
'llo'
+ >>> result = df.select(
+ ... dfn.functions.substr(dfn.col("a"), 2, length=3).alias("s"))
+ >>> result.collect_column("s")[0].as_py()
+ 'ell'
+
+ .. seealso:: :py:func:`substring`.
+
.. py:function:: substr_index(string: datafusion.expr.Expr, delimiter: datafusion.expr.Expr | str, count: datafusion.expr.Expr | int) -> datafusion.expr.Expr
@@ -5583,13 +6002,20 @@ Package Contents
'hello'
+.. py:function:: substring_index(string: datafusion.expr.Expr, delimiter: datafusion.expr.Expr | str, count: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+
+ Returns an indexed substring.
+
+ .. seealso:: This is an alias for :py:func:`substr_index`.
+
+
.. py:function:: sum(expression: datafusion.expr.Expr, distinct: bool = False, filter: datafusion.expr.Expr | None = None) -> datafusion.expr.Expr
Computes the sum of a set of numbers.
This aggregate function expects a numeric expression.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by`` and ``null_treatment``.
:param expression: Values to combine into an array
@@ -5942,9 +6368,9 @@ Package Contents
'HELLO'
-.. py:function:: trim(arg: datafusion.expr.Expr) -> datafusion.expr.Expr
+.. py:function:: trim(arg: datafusion.expr.Expr, characters: datafusion.expr.Expr | str | None = None) -> datafusion.expr.Expr
- Removes all characters, spaces by default, from both sides of a string.
+ Removes ``characters``, spaces by default, from both sides of a string.
.. rubric:: Examples
@@ -5954,6 +6380,15 @@ Package Contents
>>> result.collect_column("t")[0].as_py()
'hello'
+ Trim a different set of characters:
+
+ >>> df = ctx.from_pydict({"a": ["xxhelloxx"]})
+ >>> result = df.select(
+ ... dfn.functions.trim(dfn.col("a"), characters="x").alias("t")
+ ... )
+ >>> result.collect_column("t")[0].as_py()
+ 'hello'
+
.. py:function:: trunc(num: datafusion.expr.Expr, precision: datafusion.expr.Expr | int | None = None) -> datafusion.expr.Expr
@@ -6084,7 +6519,7 @@ Package Contents
Computes the population variance of the argument.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param expression: The variable to compute the variance for
@@ -6121,7 +6556,7 @@ Package Contents
Computes the sample variance of the argument.
- If using the builder functions described in ref:`_aggregation` this function ignores
+ If using the builder functions described in :ref:`aggregation` this function ignores
the options ``order_by``, ``null_treatment``, and ``distinct``.
:param expression: The variable to compute the variance for
diff --git a/_sources/autoapi/datafusion/functions/spark/index.rst.txt b/_sources/autoapi/datafusion/functions/spark/index.rst.txt
index d69eaa7d..cccbbd5a 100644
--- a/_sources/autoapi/datafusion/functions/spark/index.rst.txt
+++ b/_sources/autoapi/datafusion/functions/spark/index.rst.txt
@@ -30,6 +30,7 @@ Functions
datafusion.functions.spark.array_contains
datafusion.functions.spark.array_repeat
datafusion.functions.spark.ascii
+ datafusion.functions.spark.atan2
datafusion.functions.spark.avg
datafusion.functions.spark.base64
datafusion.functions.spark.bin
@@ -40,10 +41,14 @@ Functions
datafusion.functions.spark.bitmap_count
datafusion.functions.spark.bitwise_not
datafusion.functions.spark.ceil
+ datafusion.functions.spark.ceiling
datafusion.functions.spark.char
+ datafusion.functions.spark.char_length
+ datafusion.functions.spark.character_length
datafusion.functions.spark.collect_list
datafusion.functions.spark.collect_set
datafusion.functions.spark.concat
+ datafusion.functions.spark.concat_ws
datafusion.functions.spark.crc32
datafusion.functions.spark.csc
datafusion.functions.spark.date_add
@@ -51,14 +56,19 @@ Functions
datafusion.functions.spark.date_part
datafusion.functions.spark.date_sub
datafusion.functions.spark.date_trunc
+ datafusion.functions.spark.dateadd
+ datafusion.functions.spark.datediff
+ datafusion.functions.spark.datepart
datafusion.functions.spark.elt
datafusion.functions.spark.expm1
datafusion.functions.spark.factorial
datafusion.functions.spark.floor
datafusion.functions.spark.format_string
datafusion.functions.spark.from_utc_timestamp
+ datafusion.functions.spark.getbit
datafusion.functions.spark.hex
datafusion.functions.spark.hour
+ datafusion.functions.spark.hypot
datafusion.functions.spark.if_
datafusion.functions.spark.ilike
datafusion.functions.spark.is_valid_utf8
@@ -74,14 +84,20 @@ Functions
datafusion.functions.spark.map_from_entries
datafusion.functions.spark.minute
datafusion.functions.spark.modulus
+ datafusion.functions.spark.monthname
datafusion.functions.spark.negative
datafusion.functions.spark.next_day
datafusion.functions.spark.parse_url
datafusion.functions.spark.pmod
+ datafusion.functions.spark.pow
+ datafusion.functions.spark.power
+ datafusion.functions.spark.printf
+ datafusion.functions.spark.quote
datafusion.functions.spark.rint
datafusion.functions.spark.round
datafusion.functions.spark.sec
datafusion.functions.spark.second
+ datafusion.functions.spark.sha
datafusion.functions.spark.sha1
datafusion.functions.spark.sha2
datafusion.functions.spark.shiftleft
@@ -94,6 +110,7 @@ Functions
datafusion.functions.spark.space
datafusion.functions.spark.spark_cast
datafusion.functions.spark.str_to_map
+ datafusion.functions.spark.substr
datafusion.functions.spark.substring
datafusion.functions.spark.time_trunc
datafusion.functions.spark.to_utc_timestamp
@@ -109,6 +126,7 @@ Functions
datafusion.functions.spark.unix_seconds
datafusion.functions.spark.url_decode
datafusion.functions.spark.url_encode
+ datafusion.functions.spark.weekday
datafusion.functions.spark.width_bucket
datafusion.functions.spark.xxhash64
@@ -212,6 +230,22 @@ Module Contents
65
+.. py:function:: atan2(col1: datafusion.expr.Expr | float, col2: datafusion.expr.Expr | float) -> datafusion.expr.Expr
+
+ Spark ``atan2``: angle in radians of the point ``(col2, col1)``.
+
+ ``col1`` is the y coordinate and ``col2`` the x coordinate. Both accept
+ native numbers or :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.atan2(1.0, 0.0).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 1.5707963267948966
+
+
.. py:function:: avg(col: datafusion.expr.Expr, distinct: bool | None = None, filter: datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None, null_treatment: datafusion.common.NullTreatment | None = None) -> datafusion.expr.Expr
Spark ``avg``: returns the mean of a numeric column.
@@ -350,6 +384,13 @@ Module Contents
2
+.. py:function:: ceiling(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``ceiling``: smallest integer ≥ arg.
+
+ .. seealso:: This is an alias for :py:func:`ceil`.
+
+
.. py:function:: char(col: datafusion.expr.Expr) -> datafusion.expr.Expr
Spark ``char``: ASCII character for a code point (mod 256).
@@ -363,6 +404,20 @@ Module Contents
'A'
+.. py:function:: char_length(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``char_length``: character length of a string, or bytes of binary.
+
+ .. seealso:: This is an alias for :py:func:`length`.
+
+
+.. py:function:: character_length(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``character_length``: character length of a string, or bytes of binary.
+
+ .. seealso:: This is an alias for :py:func:`length`.
+
+
.. py:function:: collect_list(col: datafusion.expr.Expr, distinct: bool | None = None, filter: datafusion.expr.Expr | None = None, order_by: list[datafusion.expr.SortKey] | datafusion.expr.SortKey | None = None, null_treatment: datafusion.common.NullTreatment | None = None) -> datafusion.expr.Expr
Spark ``collect_list``: collect values into an array (preserves dups).
@@ -405,6 +460,25 @@ Module Contents
'ab'
+.. py:function:: concat_ws(sep: datafusion.expr.Expr | str, *cols: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``concat_ws``: joins strings and arrays of strings with ``sep``.
+
+ NULL inputs are skipped rather than propagated.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["x"], "b": [None], "c": ["z"]})
+ >>> r = df.select(
+ ... dfn.functions.spark.concat_ws(
+ ... "-", dfn.col("a"), dfn.col("b"), dfn.col("c")
+ ... ).alias("v")
+ ... )
+ >>> r.collect_column("v")[0].as_py()
+ 'x-z'
+
+
.. py:function:: crc32(col: datafusion.expr.Expr) -> datafusion.expr.Expr
Spark ``crc32``: cyclic redundancy check value as a bigint.
@@ -524,6 +598,27 @@ Module Contents
datetime.datetime(2020, 1, 1, 0, 0)
+.. py:function:: dateadd(start: datafusion.expr.Expr, days: datafusion.expr.Expr | int) -> datafusion.expr.Expr
+
+ Spark ``dateadd``: date + N days.
+
+ .. seealso:: This is an alias for :py:func:`date_add`.
+
+
+.. py:function:: datediff(end: datafusion.expr.Expr, start: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``datediff``: number of days from ``start`` to ``end``.
+
+ .. seealso:: This is an alias for :py:func:`date_diff`.
+
+
+.. py:function:: datepart(field: datafusion.expr.Expr | str, source: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``datepart``: extract ``field`` from a date/time/timestamp.
+
+ .. seealso:: This is an alias for :py:func:`date_part`.
+
+
.. py:function:: elt(*inputs: datafusion.expr.Expr) -> datafusion.expr.Expr
Spark ``elt``: returns the n-th input (1-indexed).
@@ -626,6 +721,13 @@ Module Contents
datetime.datetime(2020, 1, 15, 14, 30, 45)
+.. py:function:: getbit(col: datafusion.expr.Expr, pos: datafusion.expr.Expr | str) -> datafusion.expr.Expr
+
+ Spark ``getbit``: returns the bit (0 or 1) at ``pos``.
+
+ .. seealso:: This is an alias for :py:func:`bit_get`.
+
+
.. py:function:: hex(col: datafusion.expr.Expr) -> datafusion.expr.Expr
Spark ``hex``: hexadecimal representation.
@@ -657,6 +759,21 @@ Module Contents
14
+.. py:function:: hypot(col1: datafusion.expr.Expr | float, col2: datafusion.expr.Expr | float) -> datafusion.expr.Expr
+
+ Spark ``hypot``: ``sqrt(col1^2 + col2^2)`` without intermediate overflow.
+
+ Both arguments accept native numbers or :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.hypot(3.0, 4.0).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 5.0
+
+
.. py:function:: if_(condition: datafusion.expr.Expr, if_true: datafusion.expr.Expr | Any, if_false: datafusion.expr.Expr | Any) -> datafusion.expr.Expr
Spark ``if``: returns ``if_true`` when ``condition`` is true, else ``if_false``.
@@ -729,7 +846,7 @@ Module Contents
{'c0': '1', 'c1': 'x'}
-.. py:function:: last_day(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+.. py:function:: last_day(date: datafusion.expr.Expr) -> datafusion.expr.Expr
Spark ``last_day``: last day of the month containing the date.
@@ -927,6 +1044,22 @@ Module Contents
1
+.. py:function:: monthname(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``monthname``: three-letter abbreviated month name.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import date
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> d = dfn.lit(pa.scalar(date(2024, 3, 15)))
+ >>> r = df.select(dfn.functions.spark.monthname(d).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 'Mar'
+
+
.. py:function:: negative(col: datafusion.expr.Expr) -> datafusion.expr.Expr
Spark ``negative``: unary minus.
@@ -1007,6 +1140,63 @@ Module Contents
2
+.. py:function:: pow(col1: datafusion.expr.Expr | float, col2: datafusion.expr.Expr | float) -> datafusion.expr.Expr
+
+ Spark ``pow``: ``col1`` raised to the power ``col2``, as a double.
+
+ Both arguments accept native numbers or :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.pow(2, 10).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 1024.0
+
+
+.. py:function:: power(col1: datafusion.expr.Expr | float, col2: datafusion.expr.Expr | float) -> datafusion.expr.Expr
+
+ Spark ``power``: ``col1`` raised to the power ``col2``.
+
+ .. seealso:: This is an alias for :py:func:`pow`.
+
+
+.. py:function:: printf(format: datafusion.expr.Expr | str, *cols: datafusion.expr.Expr | str) -> datafusion.expr.Expr
+
+ Spark ``printf``: printf-style format string.
+
+ Unlike :py:func:`format_string`, a bare ``str`` ``format`` or ``cols`` entry
+ is treated as a column name (matching pyspark), not a literal; pass
+ :func:`~datafusion.lit` for a literal format.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": ["aa%d%s"], "b": [123], "c": ["cc"]})
+ >>> r = df.select(dfn.functions.spark.printf("a", "b", "c").alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 'aa123cc'
+
+ >>> r = df.select(
+ ... dfn.functions.spark.printf(dfn.lit("%d-%s"), "b", "c").alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ '123-cc'
+
+
+.. py:function:: quote(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``quote``: wraps a string in single quotes, escaping inner quotes.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.quote(dfn.lit("it's")).alias("v"))
+ >>> print(r.collect_column("v")[0].as_py())
+ 'it\'s'
+
+
.. py:function:: rint(col: datafusion.expr.Expr) -> datafusion.expr.Expr
Spark ``rint``: round to nearest mathematical integer (as double).
@@ -1072,6 +1262,13 @@ Module Contents
45
+.. py:function:: sha(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``sha``: SHA-1 hash as a hex string.
+
+ .. seealso:: This is an alias for :py:func:`sha1`.
+
+
.. py:function:: sha1(col: datafusion.expr.Expr) -> datafusion.expr.Expr
Spark ``sha1``: SHA-1 hash as a hex string.
@@ -1196,12 +1393,10 @@ Module Contents
.. rubric:: Examples
>>> ctx = dfn.SessionContext()
- >>> df = ctx.from_pydict({"x": [1]})
+ >>> df = ctx.from_pydict({"x": [[1, 2, 3, 4]]})
>>> r = df.select(
... dfn.functions.spark.slice(
- ... dfn.functions.spark.array(
- ... dfn.lit(1), dfn.lit(2), dfn.lit(3), dfn.lit(4)),
- ... 2, 2,
+ ... dfn.col("x"), 2, 2,
... ).alias("v")
... )
>>> r.collect_column("v")[0].as_py()
@@ -1287,6 +1482,27 @@ Module Contents
[('a', '1'), ('b', '2')]
+.. py:function:: substr(str: datafusion.expr.Expr, pos: datafusion.expr.Expr | int, len: datafusion.expr.Expr | int | None = None) -> datafusion.expr.Expr
+
+ Spark ``substr``: 1-indexed substring, to the end when ``len`` is omitted.
+
+ Same as :py:func:`substring` except that ``len`` is optional. ``pos`` and
+ ``len`` accept native ``int`` values or :class:`Expr`.
+
+ .. rubric:: Examples
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> r = df.select(dfn.functions.spark.substr(dfn.lit("hello"), 2).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 'ello'
+
+ >>> r = df.select(
+ ... dfn.functions.spark.substr(dfn.lit("hello"), 2, len=3).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 'ell'
+
+
.. py:function:: substring(str: datafusion.expr.Expr, pos: datafusion.expr.Expr | int, len: datafusion.expr.Expr | int) -> datafusion.expr.Expr
Spark ``substring``: 1-indexed substring starting at ``pos`` of given ``length``.
@@ -1536,6 +1752,22 @@ Module Contents
'a+b'
+.. py:function:: weekday(col: datafusion.expr.Expr) -> datafusion.expr.Expr
+
+ Spark ``weekday``: day of the week, Monday = 0 through Sunday = 6.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datetime import date
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"x": [1]})
+ >>> d = dfn.lit(pa.scalar(date(2024, 3, 15)))
+ >>> r = df.select(dfn.functions.spark.weekday(d).alias("v"))
+ >>> r.collect_column("v")[0].as_py()
+ 4
+
+
.. py:function:: width_bucket(v: datafusion.expr.Expr, min: datafusion.expr.Expr, max: datafusion.expr.Expr, numBucket: datafusion.expr.Expr | int) -> datafusion.expr.Expr
Spark ``width_bucket``: bucket number for ``value`` in equi-width histogram.
diff --git a/_sources/autoapi/datafusion/index.rst.txt b/_sources/autoapi/datafusion/index.rst.txt
index 8231801a..a05824f3 100644
--- a/_sources/autoapi/datafusion/index.rst.txt
+++ b/_sources/autoapi/datafusion/index.rst.txt
@@ -57,6 +57,7 @@ Submodules
/autoapi/datafusion/dataframe/index
/autoapi/datafusion/dataframe_formatter/index
/autoapi/datafusion/expr/index
+ /autoapi/datafusion/extensions/index
/autoapi/datafusion/functions/index
/autoapi/datafusion/input/index
/autoapi/datafusion/io/index
@@ -97,6 +98,7 @@ Classes
datafusion.MetricsSet
datafusion.ParquetColumnOptions
datafusion.ParquetWriterOptions
+ datafusion.PhysicalPartitioning
datafusion.RecordBatch
datafusion.RecordBatchStream
datafusion.RuntimeEnvBuilder
@@ -427,7 +429,7 @@ Package Contents
.. py:method:: children() -> list[ExecutionPlan]
- Get a list of children `ExecutionPlan` that act as inputs to this plan.
+ Get a list of children ``ExecutionPlan`` that act as inputs to this plan.
The returned list will be empty for leaf nodes such as scans, will contain a
single value for unary nodes, or two values for binary nodes (such as joins).
@@ -479,9 +481,16 @@ Package Contents
Create an ExecutionPlan from serialized protobuf bytes.
- Decoding routes through the session's installed
- `PhysicalExtensionCodec`. Tables created in memory from record
- batches are currently not supported.
+ Decoding routes through the codecs installed on ``ctx`` with
+ :py:meth:`~datafusion.SessionContext.with_physical_extension_codec`.
+
+ Unlike :py:meth:`datafusion.Expr.from_bytes`, ``ctx`` is required and
+ positional, and there is no fallback to a worker or global context.
+ ``ctx`` need share nothing with the session that encoded the plan: a
+ scan over a table registered from record batches decodes here, because
+ the batches travel inside the encoded scan.
+
+ .. seealso:: :py:meth:`to_bytes`, :py:meth:`LogicalPlan.from_bytes`.
@@ -508,9 +517,20 @@ Package Contents
Convert an ExecutionPlan into serialized protobuf bytes.
- When ``ctx`` is supplied, encoding routes through the session's
- installed `PhysicalExtensionCodec`. Tables created in memory
- from record batches are currently not supported.
+ When ``ctx`` is supplied, encoding routes through the codecs
+ installed on it with
+ :py:meth:`~datafusion.SessionContext.with_physical_extension_codec`.
+
+ Unlike :py:meth:`LogicalPlan.to_bytes`, a plan reading a table
+ registered from record batches does round-trip: the batches travel
+ inside the encoded scan.
+
+ Round-tripping through this method and :py:meth:`from_bytes` is how
+ an extension library checks that its own codec claimed its nodes,
+ rather than a codec installed earlier in the chain — see
+ :ref:`extension_codec_order`.
+
+ .. seealso:: :py:meth:`from_bytes`, :py:meth:`LogicalPlan.to_bytes`.
@@ -523,12 +543,54 @@ Package Contents
.. py:attribute:: _raw_plan
+ .. py:property:: output_partitioning
+ :type: PhysicalPartitioning
+
+
+ Returns how this plan's output rows are spread across its partitions.
+
+ Where :py:attr:`partition_count` gives only the number of partitions,
+ this also reports the scheme, so a caller executing partitions
+ separately can tell whether they are hash-distributed on known keys or
+ merely counted. A plan does not necessarily partition the way it was
+ asked to; see :ref:`checking_partitioning`.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datafusion import SessionConfig, SessionContext
+ >>> ctx = SessionContext(SessionConfig().with_target_partitions(4))
+ >>> ctx.register_record_batches("t", [
+ ... [pa.record_batch({"a": [1, 2, 3]})],
+ ... [pa.record_batch({"a": [4, 5, 6]})],
+ ... ])
+ >>> ctx.sql("select a from t").execution_plan().output_partitioning
+ UnknownPartitioning(2)
+
+ A group-by redistributes rows, so the plan reports the keys:
+
+ >>> grouped = ctx.sql("select a, count(*) from t group by a")
+ >>> partitioning = grouped.execution_plan().output_partitioning
+ >>> partitioning.scheme
+ 'Hash'
+ >>> partitioning.partition_count
+ 4
+
+
.. py:property:: partition_count
:type: int
Returns the number of partitions in the physical plan.
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> df.execution_plan().partition_count
+ 1
+
.. py:class:: ExplainFormat
@@ -709,6 +771,13 @@ Package Contents
portable across minor versions. See :meth:`to_bytes` for
details on what travels by value vs. by reference.
+ The encoding side honors a driver-side sender context installed
+ via :func:`datafusion.ipc.set_sender_ctx` — that is how
+ :meth:`SessionContext.with_python_udf_inlining` propagates
+ through ``pickle.dumps``. The sender context is read by
+ ``__reduce__``, so :func:`copy.copy` and :func:`copy.deepcopy`
+ — which also go through ``__reduce__`` — pick it up too.
+
.. rubric:: Examples
>>> import pickle
@@ -717,13 +786,6 @@ Package Contents
>>> pickle.loads(pickle.dumps(e)).canonical_name()
'a * Int64(2)'
- The encoding side honors a driver-side sender context installed
- via :func:`datafusion.ipc.set_sender_ctx` — that is how
- :meth:`SessionContext.with_python_udf_inlining` propagates
- through ``pickle.dumps``. The sender context is read by
- ``__reduce__``, so :func:`copy.copy` and :func:`copy.deepcopy`
- — which also go through ``__reduce__`` — pick it up too.
-
.. py:method:: __repr__() -> str
@@ -999,8 +1061,9 @@ Package Contents
Only evaluate distinct values for an aggregate function.
This function will create an :py:class:`ExprFuncBuilder` that can be used to
- set parameters for either window or aggregate functions. If used on any other
- type of expression, an error will be generated when ``build()`` is called.
+ set parameters for either window or aggregate functions. It raises on a window
+ function unless that is an aggregate used as one. If used on any other type of
+ expression, an error will be generated when ``build()`` is called.
@@ -1039,8 +1102,9 @@ Package Contents
Filter an aggregate function.
This function will create an :py:class:`ExprFuncBuilder` that can be used to
- set parameters for either window or aggregate functions. If used on any other
- type of expression, an error will be generated when ``build()`` is called.
+ set parameters for either window or aggregate functions. It raises on a window
+ function unless that is an aggregate used as one. If used on any other type of
+ expression, an error will be generated when ``build()`` is called.
@@ -1073,6 +1137,13 @@ Package Contents
(sufficient for built-ins and Python UDFs, plus any UDFs
registered on the global context).
+ Note that the global context carries no extension codecs, so
+ falling back to it cannot decode a payload written by one. A
+ worker that must decode extension-codec payloads has to install a
+ context carrying those codecs — see
+ :func:`datafusion.ipc.set_worker_ctx` and
+ :meth:`SessionContext.with_extensions`.
+
.. warning:: Security
Decoding may invoke ``cloudpickle.loads`` on bytes embedded
in the payload, which executes arbitrary Python code. Treat
@@ -1112,6 +1183,12 @@ Package Contents
+ .. py:method:: is_nan() -> Expr
+
+ Returns true if a given number is +NaN or -NaN otherwise returns false.
+
+
+
.. py:method:: is_not_null() -> Expr
Returns ``True`` if this expression is not null.
@@ -1259,12 +1336,19 @@ Package Contents
.. py:method:: over(window: Window) -> Expr
- Turn an aggregate function into a window function.
+ Evaluate an aggregate or window function over a window.
- This function turns any aggregate function into a window function. With the
+ On an aggregate function this turns it into a window function. With the
exception of ``partition_by``, how each of the parameters is used is determined
by the underlying aggregate function.
+ On an aggregate, the ``null_treatment``, ``filter``, and ``distinct`` options
+ it was built with are kept, and an ``order_by`` raises; see
+ :ref:`aggregate_over_options`.
+
+ On a window function that already has a ``partition_by``, ``order_by``, or
+ ``window_frame``, this raises; see :ref:`window_function_chaining`.
+
:param window: Window definition
@@ -1274,8 +1358,10 @@ Package Contents
Set the partitioning for a window function.
This function will create an :py:class:`ExprFuncBuilder` that can be used to
- set parameters for either window or aggregate functions. If used on any other
- type of expression, an error will be generated when ``build()`` is called.
+ set parameters for either window or aggregate functions. It raises on an
+ aggregate function that has not been turned into a window function with
+ :py:meth:`over`. If used on any other type of expression, an error will be
+ generated when ``build()`` is called.
@@ -1384,7 +1470,7 @@ Package Contents
- .. py:method:: sort(ascending: bool = True, nulls_first: bool = True) -> SortExpr
+ .. py:method:: sort(ascending: bool = True, nulls_first: bool = False) -> SortExpr
Creates a sort :py:class:`Expr` from an existing :py:class:`Expr`.
@@ -1569,11 +1655,13 @@ Package Contents
.. py:method:: window_frame(window_frame: WindowFrame) -> ExprFuncBuilder
- Set the frame fora window function.
+ Set the frame for a window function.
This function will create an :py:class:`ExprFuncBuilder` that can be used to
- set parameters for either window or aggregate functions. If used on any other
- type of expression, an error will be generated when ``build()`` is called.
+ set parameters for either window or aggregate functions. It raises on an
+ aggregate function that has not been turned into a window function with
+ :py:meth:`over`. If used on any other type of expression, an error will be
+ generated when ``build()`` is called.
@@ -1650,7 +1738,7 @@ Package Contents
This constructor should not be called by the end user.
- .. py:method:: __eq__(other: LogicalPlan) -> bool
+ .. py:method:: __eq__(other: object) -> bool
Test equality.
@@ -1672,9 +1760,9 @@ Package Contents
Print the graph visualization of the logical plan.
- Returns a `format`able structure that produces lines meant for graphical display
- using the `DOT` language. This format can be visualized using software from
- [`graphviz`](https://graphviz.org/)
+ Returns a formattable structure that produces lines meant for graphical
+ display using the ``DOT`` language. This format can be visualized using
+ software from `graphviz <https://graphviz.org/>`_.
@@ -1696,9 +1784,18 @@ Package Contents
Create a LogicalPlan from serialized protobuf bytes.
- Decoding routes through the session's installed
- `LogicalExtensionCodec`. Tables created in memory from record
- batches are currently not supported.
+ Decoding routes through the codecs installed on ``ctx`` with
+ :py:meth:`~datafusion.SessionContext.with_logical_extension_codec`.
+ Tables created in memory from record batches are currently not
+ supported.
+
+ Unlike :py:meth:`datafusion.Expr.from_bytes`, ``ctx`` is required and
+ positional, and there is no fallback to a worker or global context.
+
+ .. seealso::
+
+ :py:meth:`to_bytes`, :py:meth:`ExecutionPlan.from_bytes`,
+ :py:meth:`datafusion.Expr.from_bytes`.
@@ -1720,12 +1817,17 @@ Package Contents
Convert a LogicalPlan to serialized protobuf bytes.
- When ``ctx`` is supplied, encoding routes through the session's
- installed `LogicalExtensionCodec` so user FFI codecs (registered
- via :py:meth:`SessionContext.with_logical_extension_codec`) see
- the encode path. With ``ctx=None`` a default codec is used.
- Tables created in memory from record batches are currently not
- supported.
+ When ``ctx`` is supplied, encoding routes through the codecs
+ installed on it with
+ :py:meth:`~datafusion.SessionContext.with_logical_extension_codec`,
+ so extension codecs see the encode path. With ``ctx=None`` a
+ default codec is used. Tables created in memory from record
+ batches are currently not supported.
+
+ .. seealso::
+
+ :py:meth:`from_bytes`, :py:meth:`ExecutionPlan.to_bytes`,
+ :py:meth:`datafusion.Expr.to_bytes`.
@@ -2131,6 +2233,140 @@ Package Contents
+.. py:class:: PhysicalPartitioning(partitioning: datafusion._internal.PhysicalPartitioning)
+
+ How a physical plan's output rows are spread across its partitions.
+
+ Returned by :py:attr:`ExecutionPlan.output_partitioning`. This is the
+ partitioning a built plan *has*. Distinct from
+ :py:class:`datafusion.expr.Partitioning`, the *logical* partitioning a
+ ``Repartition`` node records and hands back from
+ ``partitioning_scheme()`` — a request, which the plan need not honour. See
+ :ref:`checking_partitioning`.
+
+ This constructor should not be called by the end user.
+
+
+ .. py:method:: __eq__(other: object) -> bool
+
+ Compare two partitionings by scheme, count and hash expressions.
+
+ Equality is structural, and does not mirror DataFusion's own
+ comparison of the underlying type, under which two
+ ``UnknownPartitioning`` values of the same width are unequal.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> a = ctx.from_pydict({"a": [1, 2, 3]}).execution_plan()
+ >>> b = ctx.from_pydict({"b": [4, 5, 6]}).execution_plan()
+ >>> a.output_partitioning == b.output_partitioning
+ True
+ >>> a.output_partitioning == "UnknownPartitioning(1)"
+ False
+
+
+
+ .. py:method:: __hash__() -> int
+
+ Hash the partitioning, consistently with :py:meth:`__eq__`.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> a = ctx.from_pydict({"a": [1, 2, 3]}).execution_plan()
+ >>> b = ctx.from_pydict({"b": [4, 5, 6]}).execution_plan()
+ >>> len({a.output_partitioning, b.output_partitioning})
+ 1
+
+
+
+ .. py:method:: __repr__() -> str
+
+ Print a string representation of the partitioning.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> repr(df.execution_plan().output_partitioning)
+ 'UnknownPartitioning(1)'
+
+
+
+ .. py:method:: _key() -> tuple[str, int, tuple[str, Ellipsis] | None]
+
+
+ .. py:attribute:: _raw_partitioning
+
+
+ .. py:property:: hash_expressions
+ :type: list[str] | None
+
+
+ The expressions rows are hashed on, or ``None`` for other schemes.
+
+ Physical expressions have no Python representation, so these are
+ returned in their displayed form. ``None`` covers ``"Range"`` as well,
+ whose ordering and split points this class does not expose.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datafusion import SessionConfig, SessionContext
+ >>> ctx = SessionContext(SessionConfig().with_target_partitions(4))
+ >>> ctx.register_record_batches("t", [
+ ... [pa.record_batch({"a": [1, 2, 3]})],
+ ... [pa.record_batch({"a": [4, 5, 6]})],
+ ... ])
+ >>> scan = ctx.sql("select a from t").execution_plan()
+ >>> scan.output_partitioning.hash_expressions is None
+ True
+ >>> grouped = ctx.sql("select a, count(*) from t group by a")
+ >>> grouped.execution_plan().output_partitioning.hash_expressions
+ ['a@0']
+
+
+ .. py:property:: partition_count
+ :type: int
+
+
+ The number of partitions.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> df.execution_plan().output_partitioning.partition_count
+ 1
+
+
+ .. py:property:: scheme
+ :type: Literal['RoundRobinBatch', 'Hash', 'Range', 'UnknownPartitioning']
+
+
+ Which partitioning scheme this is.
+
+ ``"UnknownPartitioning"`` means the plan knows how many partitions it
+ has but nothing about how rows are distributed between them, which is
+ the usual case for a file scan. ``"RoundRobinBatch"`` and ``"Hash"``
+ come from a repartition the optimizer inserted. ``"Range"`` only
+ appears on a plan this package did not build; see
+ :ref:`checking_partitioning`.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> df.execution_plan().output_partitioning.scheme
+ 'UnknownPartitioning'
+
+
.. py:class:: RecordBatch(record_batch: datafusion._internal.RecordBatch)
This class is essentially a wrapper for :py:class:`pa.RecordBatch`.
@@ -2383,18 +2619,60 @@ Package Contents
Create a new :py:class:`SessionConfig` with the given configuration options.
- :param config_options: Configuration options.
+ Each entry is applied as though passed to :py:meth:`set`, so the same
+ keys are rejected. See :ref:`configuration`.
+
+ :param config_options: Options to apply, keyed by fully qualified name.
+
+ :raises ValueError: If a key names no known option, or a value does not
+ parse as that option's declared type. Which of several bad
+ entries is reported is not defined.
+
+ Example usage:
+
+ >>> from datafusion import SessionConfig, SessionContext
+ >>> ctx = SessionContext(SessionConfig())
+ >>> config = SessionConfig(
+ ... config_options={"datafusion.execution.batch_size": "1024"}
+ ... )
+ >>> ctx = SessionContext(config.with_information_schema(True))
+ >>> ctx.sql(
+ ... "select value from information_schema.df_settings"
+ ... " where name = 'datafusion.execution.batch_size'"
+ ... ).collect()[0]["value"][0]
+ <pyarrow.StringScalar: '1024'>
.. py:method:: set(key: str, value: str) -> SessionConfig
- Set a configuration option.
+ Set a configuration option by its fully qualified key.
- Args:
- key: Option key.
- value: Option value.
+ Not every key that ``information_schema.df_settings`` lists can be set
+ here: the ``datafusion.runtime.*`` entries come from the runtime
+ environment rather than from the session config. See
+ :ref:`configuration`.
- :returns: A new :py:class:`SessionConfig` object with the updated setting.
+ :param key: Option key including its namespace, such as
+ ``datafusion.execution.batch_size``.
+ :param value: Option value as a string, parsed according to the type the
+ option declares.
+
+ :returns: This :py:class:`SessionConfig`, modified in place, so that calls
+ chain.
+
+ :raises ValueError: If ``key`` names no known option, or if ``value`` does
+ not parse as that option's declared type.
+
+ Example usage:
+
+ >>> from datafusion import SessionConfig, SessionContext
+ >>> config = SessionConfig().set("datafusion.execution.batch_size", "1024")
+ >>> ctx = SessionContext(config.with_information_schema(True))
+ >>> ctx.sql(
+ ... "select value from information_schema.df_settings"
+ ... " where name = 'datafusion.execution.batch_size'"
+ ... ).collect()[0]["value"][0]
+ <pyarrow.StringScalar: '1024'>
diff --git a/_sources/autoapi/datafusion/ipc/index.rst.txt b/_sources/autoapi/datafusion/ipc/index.rst.txt
index d81b3443..4c630216 100644
--- a/_sources/autoapi/datafusion/ipc/index.rst.txt
+++ b/_sources/autoapi/datafusion/ipc/index.rst.txt
@@ -173,11 +173,16 @@ Module Contents
.. rubric:: Examples
>>> from datafusion import SessionContext
- >>> from datafusion.ipc import set_sender_ctx, get_sender_ctx
+ >>> from datafusion.ipc import (
+ ... clear_sender_ctx,
+ ... get_sender_ctx,
+ ... set_sender_ctx,
+ ... )
>>> driver = SessionContext().with_python_udf_inlining(enabled=False)
>>> set_sender_ctx(driver)
>>> get_sender_ctx() is driver
True
+ >>> clear_sender_ctx()
.. py:function:: set_worker_ctx(ctx: datafusion.context.SessionContext) -> None
diff --git a/_sources/autoapi/datafusion/plan/index.rst.txt b/_sources/autoapi/datafusion/plan/index.rst.txt
index 2db4c17d..5f89f805 100644
--- a/_sources/autoapi/datafusion/plan/index.rst.txt
+++ b/_sources/autoapi/datafusion/plan/index.rst.txt
@@ -18,6 +18,7 @@ Classes
datafusion.plan.LogicalPlan
datafusion.plan.Metric
datafusion.plan.MetricsSet
+ datafusion.plan.PhysicalPartitioning
Module Contents
@@ -38,7 +39,7 @@ Module Contents
.. py:method:: children() -> list[ExecutionPlan]
- Get a list of children `ExecutionPlan` that act as inputs to this plan.
+ Get a list of children ``ExecutionPlan`` that act as inputs to this plan.
The returned list will be empty for leaf nodes such as scans, will contain a
single value for unary nodes, or two values for binary nodes (such as joins).
@@ -90,9 +91,16 @@ Module Contents
Create an ExecutionPlan from serialized protobuf bytes.
- Decoding routes through the session's installed
- `PhysicalExtensionCodec`. Tables created in memory from record
- batches are currently not supported.
+ Decoding routes through the codecs installed on ``ctx`` with
+ :py:meth:`~datafusion.SessionContext.with_physical_extension_codec`.
+
+ Unlike :py:meth:`datafusion.Expr.from_bytes`, ``ctx`` is required and
+ positional, and there is no fallback to a worker or global context.
+ ``ctx`` need share nothing with the session that encoded the plan: a
+ scan over a table registered from record batches decodes here, because
+ the batches travel inside the encoded scan.
+
+ .. seealso:: :py:meth:`to_bytes`, :py:meth:`LogicalPlan.from_bytes`.
@@ -119,9 +127,20 @@ Module Contents
Convert an ExecutionPlan into serialized protobuf bytes.
- When ``ctx`` is supplied, encoding routes through the session's
- installed `PhysicalExtensionCodec`. Tables created in memory
- from record batches are currently not supported.
+ When ``ctx`` is supplied, encoding routes through the codecs
+ installed on it with
+ :py:meth:`~datafusion.SessionContext.with_physical_extension_codec`.
+
+ Unlike :py:meth:`LogicalPlan.to_bytes`, a plan reading a table
+ registered from record batches does round-trip: the batches travel
+ inside the encoded scan.
+
+ Round-tripping through this method and :py:meth:`from_bytes` is how
+ an extension library checks that its own codec claimed its nodes,
+ rather than a codec installed earlier in the chain — see
+ :ref:`extension_codec_order`.
+
+ .. seealso:: :py:meth:`from_bytes`, :py:meth:`LogicalPlan.to_bytes`.
@@ -134,12 +153,54 @@ Module Contents
.. py:attribute:: _raw_plan
+ .. py:property:: output_partitioning
+ :type: PhysicalPartitioning
+
+
+ Returns how this plan's output rows are spread across its partitions.
+
+ Where :py:attr:`partition_count` gives only the number of partitions,
+ this also reports the scheme, so a caller executing partitions
+ separately can tell whether they are hash-distributed on known keys or
+ merely counted. A plan does not necessarily partition the way it was
+ asked to; see :ref:`checking_partitioning`.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datafusion import SessionConfig, SessionContext
+ >>> ctx = SessionContext(SessionConfig().with_target_partitions(4))
+ >>> ctx.register_record_batches("t", [
+ ... [pa.record_batch({"a": [1, 2, 3]})],
+ ... [pa.record_batch({"a": [4, 5, 6]})],
+ ... ])
+ >>> ctx.sql("select a from t").execution_plan().output_partitioning
+ UnknownPartitioning(2)
+
+ A group-by redistributes rows, so the plan reports the keys:
+
+ >>> grouped = ctx.sql("select a, count(*) from t group by a")
+ >>> partitioning = grouped.execution_plan().output_partitioning
+ >>> partitioning.scheme
+ 'Hash'
+ >>> partitioning.partition_count
+ 4
+
+
.. py:property:: partition_count
:type: int
Returns the number of partitions in the physical plan.
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> df.execution_plan().partition_count
+ 1
+
.. py:class:: LogicalPlan(plan: datafusion._internal.LogicalPlan)
@@ -158,7 +219,7 @@ Module Contents
This constructor should not be called by the end user.
- .. py:method:: __eq__(other: LogicalPlan) -> bool
+ .. py:method:: __eq__(other: object) -> bool
Test equality.
@@ -180,9 +241,9 @@ Module Contents
Print the graph visualization of the logical plan.
- Returns a `format`able structure that produces lines meant for graphical display
- using the `DOT` language. This format can be visualized using software from
- [`graphviz`](https://graphviz.org/)
+ Returns a formattable structure that produces lines meant for graphical
+ display using the ``DOT`` language. This format can be visualized using
+ software from `graphviz <https://graphviz.org/>`_.
@@ -204,9 +265,18 @@ Module Contents
Create a LogicalPlan from serialized protobuf bytes.
- Decoding routes through the session's installed
- `LogicalExtensionCodec`. Tables created in memory from record
- batches are currently not supported.
+ Decoding routes through the codecs installed on ``ctx`` with
+ :py:meth:`~datafusion.SessionContext.with_logical_extension_codec`.
+ Tables created in memory from record batches are currently not
+ supported.
+
+ Unlike :py:meth:`datafusion.Expr.from_bytes`, ``ctx`` is required and
+ positional, and there is no fallback to a worker or global context.
+
+ .. seealso::
+
+ :py:meth:`to_bytes`, :py:meth:`ExecutionPlan.from_bytes`,
+ :py:meth:`datafusion.Expr.from_bytes`.
@@ -228,12 +298,17 @@ Module Contents
Convert a LogicalPlan to serialized protobuf bytes.
- When ``ctx`` is supplied, encoding routes through the session's
- installed `LogicalExtensionCodec` so user FFI codecs (registered
- via :py:meth:`SessionContext.with_logical_extension_codec`) see
- the encode path. With ``ctx=None`` a default codec is used.
- Tables created in memory from record batches are currently not
- supported.
+ When ``ctx`` is supplied, encoding routes through the codecs
+ installed on it with
+ :py:meth:`~datafusion.SessionContext.with_logical_extension_codec`,
+ so extension codecs see the encode path. With ``ctx=None`` a
+ default codec is used. Tables created in memory from record
+ batches are currently not supported.
+
+ .. seealso::
+
+ :py:meth:`from_bytes`, :py:meth:`ExecutionPlan.to_bytes`,
+ :py:meth:`datafusion.Expr.to_bytes`.
@@ -398,3 +473,137 @@ Module Contents
Sum of spilled_rows across all partitions.
+.. py:class:: PhysicalPartitioning(partitioning: datafusion._internal.PhysicalPartitioning)
+
+ How a physical plan's output rows are spread across its partitions.
+
+ Returned by :py:attr:`ExecutionPlan.output_partitioning`. This is the
+ partitioning a built plan *has*. Distinct from
+ :py:class:`datafusion.expr.Partitioning`, the *logical* partitioning a
+ ``Repartition`` node records and hands back from
+ ``partitioning_scheme()`` — a request, which the plan need not honour. See
+ :ref:`checking_partitioning`.
+
+ This constructor should not be called by the end user.
+
+
+ .. py:method:: __eq__(other: object) -> bool
+
+ Compare two partitionings by scheme, count and hash expressions.
+
+ Equality is structural, and does not mirror DataFusion's own
+ comparison of the underlying type, under which two
+ ``UnknownPartitioning`` values of the same width are unequal.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> a = ctx.from_pydict({"a": [1, 2, 3]}).execution_plan()
+ >>> b = ctx.from_pydict({"b": [4, 5, 6]}).execution_plan()
+ >>> a.output_partitioning == b.output_partitioning
+ True
+ >>> a.output_partitioning == "UnknownPartitioning(1)"
+ False
+
+
+
+ .. py:method:: __hash__() -> int
+
+ Hash the partitioning, consistently with :py:meth:`__eq__`.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> a = ctx.from_pydict({"a": [1, 2, 3]}).execution_plan()
+ >>> b = ctx.from_pydict({"b": [4, 5, 6]}).execution_plan()
+ >>> len({a.output_partitioning, b.output_partitioning})
+ 1
+
+
+
+ .. py:method:: __repr__() -> str
+
+ Print a string representation of the partitioning.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> repr(df.execution_plan().output_partitioning)
+ 'UnknownPartitioning(1)'
+
+
+
+ .. py:method:: _key() -> tuple[str, int, tuple[str, Ellipsis] | None]
+
+
+ .. py:attribute:: _raw_partitioning
+
+
+ .. py:property:: hash_expressions
+ :type: list[str] | None
+
+
+ The expressions rows are hashed on, or ``None`` for other schemes.
+
+ Physical expressions have no Python representation, so these are
+ returned in their displayed form. ``None`` covers ``"Range"`` as well,
+ whose ordering and split points this class does not expose.
+
+ .. rubric:: Examples
+
+ >>> import pyarrow as pa
+ >>> from datafusion import SessionConfig, SessionContext
+ >>> ctx = SessionContext(SessionConfig().with_target_partitions(4))
+ >>> ctx.register_record_batches("t", [
+ ... [pa.record_batch({"a": [1, 2, 3]})],
+ ... [pa.record_batch({"a": [4, 5, 6]})],
+ ... ])
+ >>> scan = ctx.sql("select a from t").execution_plan()
+ >>> scan.output_partitioning.hash_expressions is None
+ True
+ >>> grouped = ctx.sql("select a, count(*) from t group by a")
+ >>> grouped.execution_plan().output_partitioning.hash_expressions
+ ['a@0']
+
+
+ .. py:property:: partition_count
+ :type: int
+
+
+ The number of partitions.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> df.execution_plan().output_partitioning.partition_count
+ 1
+
+
+ .. py:property:: scheme
+ :type: Literal['RoundRobinBatch', 'Hash', 'Range', 'UnknownPartitioning']
+
+
+ Which partitioning scheme this is.
+
+ ``"UnknownPartitioning"`` means the plan knows how many partitions it
+ has but nothing about how rows are distributed between them, which is
+ the usual case for a file scan. ``"RoundRobinBatch"`` and ``"Hash"``
+ come from a repartition the optimizer inserted. ``"Range"`` only
+ appears on a plan this package did not build; see
+ :ref:`checking_partitioning`.
+
+ .. rubric:: Examples
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> df = ctx.from_pydict({"a": [1, 2, 3]})
+ >>> df.execution_plan().output_partitioning.scheme
+ 'UnknownPartitioning'
+
+
diff --git a/_sources/autoapi/datafusion/user_defined/index.rst.txt b/_sources/autoapi/datafusion/user_defined/index.rst.txt
index 921d97d4..b0b9ca77 100644
--- a/_sources/autoapi/datafusion/user_defined/index.rst.txt
+++ b/_sources/autoapi/datafusion/user_defined/index.rst.txt
@@ -142,13 +142,13 @@ Module Contents
Wrap an already-constructed internal ``AggregateUDF`` handle.
- Used by :py:meth:`SessionContext.udaf` to surface a function looked
- up from the session's function registry without re-running
- :py:meth:`__init__`.
+ Used by :py:meth:`SessionContext.udaf` and :py:meth:`from_pycapsule`
+ to wrap a handle from the session's function registry or an FFI
+ capsule without re-running :py:meth:`__init__`.
- .. py:method:: from_pycapsule(func: AggregateUDFExportable | _typeshed.CapsuleType) -> AggregateUDF
+ .. py:method:: from_pycapsule(func: AggregateUDFExportable | types.CapsuleType) -> AggregateUDF
:staticmethod:
@@ -162,7 +162,7 @@ Module Contents
.. py:method:: udaf(input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str, name: str | None = None) -> collections.abc.Callable[Ellipsis, AggregateUDF]
udaf(accum: collections.abc.Callable[[], Accumulator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str, name: str | None = None) -> AggregateUDF
udaf(accum: AggregateUDFExportable) -> AggregateUDF
- udaf(accum: _typeshed.CapsuleType) -> AggregateUDF
+ udaf(accum: types.CapsuleType) -> AggregateUDF
:staticmethod:
@@ -292,8 +292,56 @@ Module Contents
Type hint for objects exposing ``__datafusion_logical_extension_codec__``.
+ Implement the codec itself exactly as you would for a session that installs
+ only yours. A session may hold several codecs, but each payload records the
+ codec that wrote it and is only ever handed back to that codec, so there is
+ no need to recognise or reject another library's payloads.
- .. py:method:: __datafusion_logical_extension_codec__() -> object
+ A serialized plan records which codec wrote each payload, as a short id
+ taken from your class's module and qualified name. An optional
+ ``__datafusion_codec_id__`` attribute pins that id instead. It is not part
+ of this protocol and is rarely needed: declare it when renaming your class
+ must not stop older plans from decoding, or when one library installs two
+ instances that own disjoint slices of the wire format.
+
+ :param session: A handle on the session this codec is being installed on. Take
+ the task context provider from it rather than building a session of
+ your own, so the decode callbacks resolve names against the session
+ that runs the query.
+
+ **Duck-type it; do not check its type.** This argument is the
+ reference for every capsule getter in the protocol, and across them
+ it is not always a session at all: the provider and catalog getters
+ receive the host's logical codec as a bare ``PyCapsule``. Even when
+ it *is* a session it is the PyO3 context from
+ ``datafusion._internal``, not the
+ :py:class:`~datafusion.context.SessionContext` wrapper, so it
+ exposes every capsule getter and ``__datafusion_codec_id__`` —
+ everything the protocol asks of it — but
+ ``isinstance(session, SessionContext)`` is ``False`` even though its
+ ``repr`` reads ``datafusion.SessionContext``. Pass it to
+ ``ffi_logical_codec_from_pycapsule``, which handles both.
+
+ The two bundle hooks are the exception:
+ :py:class:`~datafusion.extensions.SessionComponentsExportable` and
+ :py:class:`~datafusion.extensions.SessionPlannerExportable` are
+ dispatched from Python and receive the wrapper. See
+ :ref:`extension_getter_argument`.
+
+ :returns: A ``datafusion_logical_extension_codec`` PyCapsule.
+
+ .. rubric:: Examples
+
+ A :py:class:`~datafusion.context.SessionContext` satisfies this
+ protocol, which is how you obtain a real capsule to test against:
+
+ >>> from datafusion import SessionContext
+ >>> ctx = SessionContext()
+ >>> type(ctx.__datafusion_logical_extension_codec__(ctx)).__name__
+ 'PyCapsule'
+
+
+ .. py:method:: __datafusion_logical_extension_codec__(session: Any) -> object
.. py:class:: PhysicalExtensionCodecExportable
@@ -303,11 +351,16 @@ Module Contents
Type hint for objects exposing ``__datafusion_physical_extension_codec__``.
+ See :py:class:`LogicalExtensionCodecExportable` for ``session``, for why a
+ codec need not recognise other libraries' payloads, and for
+ ``__datafusion_codec_id__``.
+
- .. py:method:: __datafusion_physical_extension_codec__() -> object
+ .. py:method:: __datafusion_physical_extension_codec__(session: Any) -> object
.. py:class:: ScalarUDF(name: str, func: collections.abc.Callable[Ellipsis, _R], input_fields: list[pyarrow.Field], return_field: pyarrow.Field, volatility: Volatility | str)
+ ScalarUDF(name: str, func: ScalarUDFExportable, input_fields: None = ..., return_field: None = ..., volatility: None = ...)
Class for performing scalar user-defined functions (UDF).
@@ -340,13 +393,13 @@ Module Contents
Wrap an already-constructed internal ``ScalarUDF`` handle.
- Used by :py:meth:`SessionContext.udf` to surface a function looked
- up from the session's function registry without re-running
- :py:meth:`__init__`.
+ Used by :py:meth:`SessionContext.udf` and :py:meth:`from_pycapsule`
+ to wrap a handle from the session's function registry or an FFI
+ capsule without re-running :py:meth:`__init__`.
- .. py:method:: from_pycapsule(func: ScalarUDFExportable) -> ScalarUDF
+ .. py:method:: from_pycapsule(func: ScalarUDFExportable | types.CapsuleType) -> ScalarUDF
:staticmethod:
@@ -360,6 +413,7 @@ Module Contents
.. py:method:: udf(input_fields: collections.abc.Sequence[pyarrow.DataType | pyarrow.Field] | pyarrow.DataType | pyarrow.Field, return_field: pyarrow.DataType | pyarrow.Field, volatility: Volatility | str, name: str | None = None) -> collections.abc.Callable[Ellipsis, ScalarUDF]
udf(func: collections.abc.Callable[Ellipsis, _R], input_fields: collections.abc.Sequence[pyarrow.DataType | pyarrow.Field] | pyarrow.DataType | pyarrow.Field, return_field: pyarrow.DataType | pyarrow.Field, volatility: Volatility | str, name: str | None = None) -> ScalarUDF
udf(func: ScalarUDFExportable) -> ScalarUDF
+ udf(func: types.CapsuleType) -> ScalarUDF
:staticmethod:
@@ -387,8 +441,9 @@ Module Contents
of the arguments to ``func``. This list must be of the same length
as the number of arguments.
:type input_fields: list[pa.Field | pa.DataType]
- :param return_field: The field of the return value from the function.
- :type return_field: _R
+ :param return_field: The field of the return value
+ from the function.
+ :type return_field: pa.DataType | pa.Field
:param volatility: See `Volatility` for allowed values.
:type volatility: Volatility | str
:param name: A descriptive name for the function.
@@ -516,7 +571,7 @@ Module Contents
- .. py:method:: _create_table_udf_decorator(name: str | None = None, *, with_session: bool = False) -> collections.abc.Callable[[collections.abc.Callable[Ellipsis, Any]], TableFunction]
+ .. py:method:: _create_table_udf_decorator(name: str, *, with_session: bool = False) -> collections.abc.Callable[[collections.abc.Callable[Ellipsis, Any]], TableFunction]
:staticmethod:
@@ -769,6 +824,7 @@ Module Contents
.. py:class:: WindowUDF(name: str, func: collections.abc.Callable[[], WindowEvaluator], input_types: list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str)
+ WindowUDF(name: str, func: WindowUDFExportable, input_types: None = ..., return_type: None = ..., volatility: None = ...)
Class for performing window user-defined functions (UDF).
@@ -818,9 +874,9 @@ Module Contents
Wrap an already-constructed internal ``WindowUDF`` handle.
- Used by :py:meth:`SessionContext.udwf` to surface a function looked
- up from the session's function registry without re-running
- :py:meth:`__init__`.
+ Used by :py:meth:`SessionContext.udwf` and :py:meth:`from_pycapsule`
+ to wrap a handle from the session's function registry or an FFI
+ capsule without re-running :py:meth:`__init__`.
@@ -840,7 +896,7 @@ Module Contents
- .. py:method:: from_pycapsule(func: WindowUDFExportable) -> WindowUDF
+ .. py:method:: from_pycapsule(func: WindowUDFExportable | types.CapsuleType) -> WindowUDF
:staticmethod:
@@ -853,6 +909,8 @@ Module Contents
.. py:method:: udwf(input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: str | None = None) -> collections.abc.Callable[Ellipsis, WindowUDF]
udwf(func: collections.abc.Callable[[], WindowEvaluator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, name: str | None = None) -> WindowUDF
+ udwf(func: WindowUDFExportable) -> WindowUDF
+ udwf(func: types.CapsuleType) -> WindowUDF
:staticmethod:
@@ -944,7 +1002,7 @@ Module Contents
.. py:method:: __datafusion_window_udf__() -> object
-.. py:function:: _is_pycapsule(value: object) -> TypeGuard[_typeshed.CapsuleType]
+.. py:function:: _is_pycapsule(value: object) -> TypeGuard[types.CapsuleType]
Return ``True`` when ``value`` is a CPython ``PyCapsule``.
diff --git a/_sources/contributor-guide/ffi-internals.md.txt b/_sources/contributor-guide/ffi-internals.md.txt
new file mode 100644
index 00000000..bbf6b65e
--- /dev/null
+++ b/_sources/contributor-guide/ffi-internals.md.txt
@@ -0,0 +1,185 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(ffi_internals)=
+
+# FFI framing internals
+
+Read this before changing how datafusion-python frames FFI components. You do
+**not** need it to write an extension library — that is the
+{ref}`Extension Guide <extension_guide>`.
+
+The invariants here are the reasons the extension-facing rules are shaped the
+way they are. Each one has an "obvious" simplification that does not work, and
+the point of this page is to record why, so the next person does not spend a
+week rediscovering it.
+
+For the wire format itself — how a codec id is stored alongside a payload,
+routed back on decode, and which two cases stay unframed — see the module
+documentation in `crates/core/src/codec.rs`. It is the authority; this page
+does not restate it.
+
+(ffi_internals_one_arc)=
+
+## One session, one `Arc<SessionContext>`
+
+Every codec handed to a foreign object carries an `FFI_TaskContextProvider`,
+and that type holds its provider **weakly**. A registered catalog provider
+upgrades the handle on every `supports_filters_pushdown` and every `scan`.
+Those handles are bound to one particular `Arc<SessionContext>` allocation, not
+to the logical session, so anything that replaces the allocation orphans all of
+them and the next query fails with `TaskContextProvider went out of scope over
+FFI boundary`.
+
+So a `PySessionContext` keeps the `Arc<SessionContext>` it was created with for
+its whole life. Installing a query planner writes the new `SessionState` back
+through `state_ref()`, exactly as `add_physical_optimizer_rule` does, rather
+than deriving a replacement context. The session id is carried across that
+rewrite — `SessionStateBuilder` mints a fresh one otherwise — so `session_id()`
+and every `TaskContext` the session hands out keep agreeing.
+
+Repairing the damage instead of avoiding it does not work in general. A context
+can rebuild the codecs it holds in its own fields, but a codec already embedded
+in a registered `FFI_CatalogProvider` — and in every `FFI_SchemaProvider` and
+`FFI_TableProvider` minted from it — is not reachable from Python at all. Nor
+can the codec simply retain the session that built it: a codec handed to a
+provider is routinely registered straight back into that same session, which
+would close the cycle
+`SessionContext -> catalog -> FFI provider -> FFI codec -> SessionContext` and
+leak it.
+
+`SessionContext.enable_url_table` is the one exception. It clones the
+underlying `SessionContext`, so the returned context has an allocation of its
+own and must not outlive the receiver. It also forks the session's state while
+keeping its id, so two handles report one `session_id()` with divergent
+configuration. That is a bug rather than a design, tracked in
+[apache/datafusion-python#1708](https://github.com/apache/datafusion-python/issues/1708);
+do not copy the pattern.
+
+(ffi_internals_rebinding)=
+
+## Why planner codec rebinding is one level deep
+
+Installing a codec on a session that already has a foreign planner rebuilds
+that planner against the new chain. The rebuild swaps the codecs on the
+installed `ForeignQueryPlanner` handle, and only that handle. A planner that
+wraps a fallback resolved that fallback when *it* was installed, and holds the
+result inside its own library's private data — behind a `create_physical_plan`
+function pointer, with no Python-side handle. A codec installed afterwards
+therefore reaches the outer planner and not the fallback, which keeps whichever
+codecs were in force when it was imported.
+
+Neither side can repair that:
+
+- **The host cannot reach it.** `FFI_QueryPlanner::new_with_ffi_codecs` unwraps
+ exactly one `ForeignQueryPlanner` layer. There is no deeper handle to unwrap
+ — the same situation as a codec embedded in a registered
+ `FFI_CatalogProvider`.
+- **The planner library cannot re-derive it.** `FFI_QueryPlanner` holds its
+ codecs by value, and `Session` exposes no accessor for the ones the host
+ currently has, so `create_physical_plan` cannot pick them up from the session
+ it is handed. The rebuild has to be eager, and an eager rebuild only sees the
+ top layer.
+
+A fix has to come from upstream, and is tracked in
+[apache/datafusion#24762](https://github.com/apache/datafusion/issues/24762).
+
+The stale codecs stay usable rather than dangling — they hold weak handles to
+the one `Arc<SessionContext>` that the previous section keeps alive — so the
+effect is a fallback hop serializing with an older codec, not a failure. It is
+also invisible to the examples in this repository, which use one fallback in
+the same cdylib as its wrapper; `datafusion-ffi` short-circuits a same-library
+hop rather than serializing, so no codec runs. A fallback in a *different*
+library would serialize, and would do it with the codecs it was imported with.
+
+The extension-facing consequence — install codecs before a layered planner, and
+prefer `with_extensions` — is documented at {ref}`planner_codec_rebinding`.
+
+(ffi_internals_commit_order)=
+
+## Why `with_extensions` commits last
+
+`with_extensions` promises that a bundle which raises leaves the session as it
+was, apart from anything a hook writes to the context it is handed
+({ref}`extension_bundles_transaction`). Keeping that promise is an ordering constraint on the implementation, not
+a property of any one step, because the planner is bound on the shared
+`SessionState` rather than on the returned handle.
+
+A call therefore splits into four steps, of which only the last writes:
+
+1. **Collect.** Every `__datafusion_session_components__` runs and its codecs
+ are gathered. Nothing is installed yet, so a hook that raises here has
+ touched nothing.
+2. **Chains.** The codecs are assembled into the returned handle. Codec chains
+ live on that handle rather than on the session, so this step writes nothing
+ to the session even though it can fail on a bad capsule or a duplicate id.
+3. **Resolve.** Every `__datafusion_session_planner__` runs, in argument order,
+ against the completed chains, and each supplied planner is exported to a
+ capsule. Every hook that can raise has run by the end of this step.
+4. **Commit.** The accumulated planner is re-imported from its capsule and
+ bound, in a single `SessionState` rebuild. The bind is skipped entirely when
+ the call installed nothing, so an empty call does not drag a planner sitting
+ on another handle's codecs onto this one's.
+
+Only step 4 touches the session, and it is not itself infallible:
+`_install_extension_planner` runs `ffi_query_planner_from_pycapsule` before it
+calls `set_session_query_planner`, which cannot fail. The property that keeps
+the promise is therefore about order, not about any step being incapable of
+raising — every fallible operation, including the ones inside the commit,
+completes before the first write.
+
+That is the rule for the next field added to `SessionExtensionComponents`, not
+only a description of the current code: a new kind of component must do its
+fallible work — importing a capsule, resolving a name — before anything is
+written, so no failure can leave the session half-updated.
+
+There would be nothing to roll back to if one did. The returned handle shares one
+session with the receiver, so the damage is visible from every other handle;
+and undoing a registration is not the same as restoring what it displaced,
+because deregistering a function that shadowed a built-in removes the built-in
+too. The split is cheaper than an undo log that cannot be written correctly.
+
+The extension-facing statement of this is
+{ref}`extension_bundles_transaction`, which says only that declaring a
+component is safe where registering one during the hook is not.
+
+## Two argument kinds for one convention
+
+`CapsuleGetterArg` in `crates/util/src/lib.rs` distinguishes three cases: no
+argument, the session, and the host's logical extension codec as a bare
+capsule. Provider and catalog registration passes the codec; the extension
+codecs, the query planner, and table functions get the session.
+
+The distinction exists so that a `TypeError` from a getter that refused its
+argument can be rewritten into an `ImportError` naming what the getter *should*
+accept. Getting that message wrong sends an extension author to fix the wrong
+signature, which is why every capsule getter routes through
+`call_capsule_getter` rather than calling `getattr` directly. Three importers
+previously each had their own copy of that logic and each missed later
+corrections to it.
+
+When adding a hook, decide which arm it needs by what its FFI constructor
+requires: a task-context provider means it must have the session, since you
+cannot get a provider off a capsule. A codec alone means either works, and
+passing the codec keeps the host from handing out session handles it does not
+need to.
+
+The extension-facing statement of this is {ref}`extension_getter_argument`,
+which deliberately describes the argument by capability — "something you can
+read the host's logical codec off" — rather than by type.
diff --git a/_sources/contributor-guide/ffi.md.txt b/_sources/contributor-guide/ffi.md.txt
deleted file mode 100644
index bf65cad2..00000000
--- a/_sources/contributor-guide/ffi.md.txt
+++ /dev/null
@@ -1,259 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-(ffi)=
-
-# Python Extensions
-
-The DataFusion in Python project is designed to allow users to extend its functionality in a few core
-areas. Ideally many users would like to package their extensions as a Python package and easily
-integrate that package with this project. This page serves to describe some of the challenges we face
-when doing these integrations and the approach our project uses.
-
-## The Primary Issue
-
-Suppose you wish to use DataFusion and you have a custom data source that can produce tables that
-can then be queried against, similar to how you can register a {ref}`CSV <io_csv>` or
-{ref}`Parquet <io_parquet>` file. In DataFusion terminology, you likely want to implement a
-{ref}`Custom Table Provider <io_custom_table_provider>`. In an effort to make your data source
-as performant as possible and to utilize the features of DataFusion, you may decide to write
-your source in Rust and then expose it through [PyO3](https://pyo3.rs) as a Python library.
-
-At first glance, it may appear the best way to do this is to add the `datafusion-python`
-crate as a dependency, provide a `PyTable`, and then to register it with the
-`SessionContext`. Unfortunately, this will not work.
-
-When you produce your code as a Python library and it needs to interact with the DataFusion
-library, at the lowest level they communicate through an Application Binary Interface (ABI).
-The acronym sounds similar to API (Application Programming Interface), but it is distinctly
-different.
-
-The ABI sets the standard for how these libraries can share data and functions between each
-other. One of the key differences between Rust and other programming languages is that Rust
-does not have a stable ABI. What this means in practice is that if you compile a Rust library
-with one version of the `rustc` compiler and I compile another library to interface with it
-but I use a different version of the compiler, there is no guarantee the interface will be
-the same.
-
-In practice, this means that a Python library built with `datafusion-python` as a Rust
-dependency will generally **not** be compatible with the DataFusion Python package, even
-if they reference the same version of `datafusion-python`. If you attempt to do this, it may
-work on your local computer if you have built both packages with the same optimizations.
-This can sometimes lead to a false expectation that the code will work, but it frequently
-breaks the moment you try to use your package against the released packages.
-
-You can find more information about the Rust ABI in their
-[online documentation](https://doc.rust-lang.org/reference/abi.html).
-
-## The FFI Approach
-
-Rust supports interacting with other programming languages through it's Foreign Function
-Interface (FFI). The advantage of using the FFI is that it enables you to write data structures
-and functions that have a stable ABI. The allows you to use Rust code with C, Python, and
-other languages. In fact, the [PyO3](https://pyo3.rs) library uses the FFI to share data
-and functions between Python and Rust.
-
-The approach we are taking in the DataFusion in Python project is to incrementally expose
-more portions of the DataFusion project via FFI interfaces. This allows users to write Rust
-code that does **not** require the `datafusion-python` crate as a dependency, expose their
-code in Python via PyO3, and have it interact with the DataFusion Python package.
-
-Early adopters of this approach include [delta-rs](https://delta-io.github.io/delta-rs/)
-who has adapted their Table Provider for use in `` `datafusion-python` `` with only a few lines
-of code. Also, the DataFusion Python project uses the existing definitions from
-[Apache Arrow CStream Interface](https://arrow.apache.org/docs/format/CStreamInterface.html)
-to support importing **and** exporting tables. Any Python package that supports reading
-the Arrow C Stream interface can work with DataFusion Python out of the box! You can read
-more about working with Arrow sources in the {ref}`Data Sources <user_guide_data_sources>`
-page.
-
-To learn more about the Foreign Function Interface in Rust, the
-[Rustonomicon](https://doc.rust-lang.org/nomicon/ffi.html) is a good resource.
-
-## Inspiration from Arrow
-
-DataFusion is built upon [Apache Arrow](https://arrow.apache.org/). The canonical Python
-Arrow implementation, [pyarrow](https://arrow.apache.org/docs/python/index.html) provides
-an excellent way to share Arrow data between Python projects without performing any copy
-operations on the data. They do this by using a well defined set of interfaces. You can
-find the details about their stream interface
-[here](https://arrow.apache.org/docs/format/CStreamInterface.html). The
-[Rust Arrow Implementation](https://github.com/apache/arrow-rs) also supports these
-`C` style definitions via the Foreign Function Interface.
-
-In addition to using these interfaces to transfer Arrow data between libraries, `pyarrow`
-goes one step further to make sharing the interfaces easier in Python. They do this
-by exposing PyCapsules that contain the expected functionality.
-
-You can learn more about PyCapsules from the official
-[Python online documentation](https://docs.python.org/3/c-api/capsule.html). PyCapsules
-have excellent support in PyO3 already. The
-[PyO3 online documentation](https://pyo3.rs/main/doc/pyo3/types/struct.pycapsule) is a good source
-for more details on using PyCapsules in Rust.
-
-Two lessons we leverage from the Arrow project in DataFusion Python are:
-
-- We reuse the existing Arrow FFI functionality wherever possible.
-- We expose PyCapsules that contain a FFI stable struct.
-
-## Implementation Details
-
-The bulk of the code necessary to perform our FFI operations is in the upstream
-[DataFusion](https://datafusion.apache.org/) core repository. You can review the code and
-documentation in the [datafusion-ffi] crate.
-
-Our FFI implementation is narrowly focused at sharing data and functions with Rust backed
-libraries. This allows us to use the [abi_stable crate](https://crates.io/crates/abi_stable).
-This is an excellent crate that allows for easy conversion between Rust native types
-and FFI-safe alternatives. For example, if you needed to pass a `Vec<String>` via FFI,
-you can simply convert it to a `RVec<RString>` in an intuitive manner. It also supports
-features like `RResult` and `ROption` that do not have an obvious translation to a
-C equivalent.
-
-The [datafusion-ffi] crate has been designed to make it easy to convert from DataFusion
-traits into their FFI counterparts. For example, if you have defined a custom
-[TableProvider](https://docs.rs/datafusion/45.0.0/datafusion/catalog/trait.TableProvider.html)
-and you want to create a sharable FFI counterpart, you could write:
-
-```rust
-let my_provider = MyTableProvider::default();
-let ffi_provider = FFI_TableProvider::new(Arc::new(my_provider), false, None);
-```
-
-(ffi_pyclass_mutability)=
-
-## PyO3 class mutability guidelines
-
-PyO3 bindings should present immutable wrappers whenever a struct stores shared or
-interior-mutable state. In practice this means that any `#[pyclass]` containing an
-`Arc<RwLock<_>>` or similar synchronized primitive must opt into `#[pyclass(frozen)]`
-unless there is a compelling reason not to.
-
-The execution context illustrates the preferred pattern. `PySessionContext` in
-{file}`src/context.rs` stays frozen even though it shares mutable state internally via
-`SessionContext`. This ensures PyO3 tracks borrows correctly while Python-facing APIs
-clone the inner `SessionContext` or return new wrappers instead of mutating the
-existing instance in place:
-
-```rust
-#[pyclass(from_py_object, frozen, name = "SessionContext", module = "datafusion", subclass)]
-#[derive(Clone)]
-pub struct PySessionContext {
- pub ctx: SessionContext,
-}
-```
-
-Occasionally a type must remain mutable—for example when PyO3 attribute setters need to
-update fields directly. In these rare cases add an inline justification so reviewers and
-future contributors understand why `frozen` is unsafe to enable. `DataTypeMap` in
-{file}`src/common/data_type.rs` includes such a comment because PyO3 still needs to track
-field updates:
-
-```rust
-// TODO: This looks like this needs pyo3 tracking so leaving unfrozen for now
-#[derive(Debug, Clone)]
-#[pyclass(from_py_object, name = "DataTypeMap", module = "datafusion.common", subclass)]
-pub struct DataTypeMap {
- #[pyo3(get, set)]
- pub arrow_type: PyDataType,
- #[pyo3(get, set)]
- pub python_type: PythonType,
- #[pyo3(get, set)]
- pub sql_type: SqlType,
-}
-```
-
-When reviewers encounter a mutable `#[pyclass]` without a comment, they should request
-an explanation or ask that `frozen` be added. Keeping these wrappers frozen by default
-helps avoid subtle bugs stemming from PyO3's interior mutability tracking.
-
-If you were interfacing with a library that provided the above `FFI_TableProvider` and
-you needed to turn it back into an `TableProvider`, you can turn it into a
-`ForeignTableProvider` with implements the `TableProvider` trait.
-
-```rust
-let foreign_provider: ForeignTableProvider = ffi_provider.into();
-```
-
-If you review the code in [datafusion-ffi] you will find that each of the traits we share
-across the boundary has two portions, one with a `FFI_` prefix and one with a `Foreign`
-prefix. This is used to distinguish which side of the FFI boundary that struct is
-designed to be used on. The structures with the `FFI_` prefix are to be used on the
-**provider** of the structure. In the example we're showing, this means the code that has
-written the underlying `TableProvider` implementation to access your custom data source.
-The structures with the `Foreign` prefix are to be used by the receiver. In this case,
-it is the `datafusion-python` library.
-
-In order to share these FFI structures, we need to wrap them in some kind of Python object
-that can be used to interface from one package to another. As described in the above
-section on our inspiration from Arrow, we use `PyCapsule`. We can create a `PyCapsule`
-for our provider thusly:
-
-```rust
-let name = CString::new("datafusion_table_provider")?;
-let my_capsule = PyCapsule::new_bound(py, provider, Some(name))?;
-```
-
-On the receiving side, turn this pycapsule object into the `FFI_TableProvider`, which
-can then be turned into a `ForeignTableProvider` the associated code is:
-
-```rust
-let capsule = capsule.cast::<PyCapsule>()?;
-let data: NonNull<FFI_TableProvider> = capsule
- .pointer_checked(Some(name))?
- .cast();
-let codec = unsafe { data.as_ref() };
-```
-
-By convention the `datafusion-python` library expects a Python object that has a
-`TableProvider` PyCapsule to have this capsule accessible by calling a function named
-`__datafusion_table_provider__`. You can see a complete working example of how to
-share a `TableProvider` from one python library to DataFusion Python in the
-[repository examples folder](https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-example).
-
-This section has been written using `TableProvider` as an example. It is the first
-extension that has been written using this approach and the most thoroughly implemented.
-As we continue to expose more of the DataFusion features, we intend to follow this same
-design pattern.
-
-## Alternative Approach
-
-Suppose you needed to expose some other features of DataFusion and you could not wait
-for the upstream repository to implement the FFI approach we describe. In this case
-you decide to create your dependency on the `datafusion-python` crate instead.
-
-As we discussed, this is not guaranteed to work across different compiler versions and
-optimization levels. If you wish to go down this route, there are two approaches we
-have identified you can use.
-
-1. Re-export all of `datafusion-python` yourself with your extensions built in.
-2. Carefully synchronize your software releases with the `datafusion-python` CI build
- system so that your libraries use the exact same compiler, features, and
- optimization level.
-
-We currently do not recommend either of these approaches as they are difficult to
-maintain over a long period. Additionally, they require a tight version coupling
-between libraries.
-
-## Status of Work
-
-At the time of this writing, the FFI features are under active development. To see
-the latest status, we recommend reviewing the code in the [datafusion-ffi] crate.
-
-[datafusion-ffi]: https://crates.io/crates/datafusion-ffi
diff --git a/_sources/contributor-guide/index.md.txt b/_sources/contributor-guide/index.md.txt
index a989a068..45787f98 100644
--- a/_sources/contributor-guide/index.md.txt
+++ b/_sources/contributor-guide/index.md.txt
@@ -19,11 +19,17 @@
# Contributor Guide
-Guides for contributors to the DataFusion in Python project.
+Guides for contributors to the DataFusion in Python project — people changing
+this repository itself.
+
+If you are writing a separate library that plugs into datafusion-python, you
+want the {ref}`Extension Guide <extension_guide>` instead. If you are using
+one, see {ref}`user_guide_extensions`.
```{toctree}
:maxdepth: 2
introduction
-ffi
+pyo3-guidelines
+ffi-internals
```
diff --git a/_sources/contributor-guide/introduction.md.txt b/_sources/contributor-guide/introduction.md.txt
index 683b27bf..24d56ed9 100644
--- a/_sources/contributor-guide/introduction.md.txt
+++ b/_sources/contributor-guide/introduction.md.txt
@@ -47,6 +47,8 @@ Bootstrap:
```shell
# fetch this repo
git clone git@github.com:apache/datafusion-python.git
+# cd to the repo root
+cd datafusion-python/
# create the virtual environment
uv sync --dev --no-install-package datafusion
# activate the environment
@@ -64,7 +66,7 @@ Whenever rust code changes (your changes or via `git pull`):
```shell
# make sure you activate the venv using "source .venv/bin/activate" first
-maturin develop -uv
+maturin develop --uv
python -m pytest
```
diff --git a/_sources/contributor-guide/pyo3-guidelines.md.txt b/_sources/contributor-guide/pyo3-guidelines.md.txt
new file mode 100644
index 00000000..4f81e569
--- /dev/null
+++ b/_sources/contributor-guide/pyo3-guidelines.md.txt
@@ -0,0 +1,74 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(pyo3_guidelines)=
+
+# PyO3 binding guidelines
+
+Conventions for the `#[pyclass]` bindings in `crates/`. These are review
+policy for this repository, not part of the extension protocol — an extension
+library is free to shape its own classes differently.
+
+(ffi_pyclass_mutability)=
+
+## Class mutability
+
+PyO3 bindings should present immutable wrappers whenever a struct stores shared
+or interior-mutable state. In practice this means that any `#[pyclass]`
+containing an `Arc<RwLock<_>>` or similar synchronized primitive must opt into
+`#[pyclass(frozen)]` unless there is a compelling reason not to.
+
+The execution context illustrates the preferred pattern. `PySessionContext` in
+{file}`src/context.rs` stays frozen even though it shares mutable state
+internally via `SessionContext`. This ensures PyO3 tracks borrows correctly
+while Python-facing APIs clone the inner `SessionContext` or return new
+wrappers instead of mutating the existing instance in place:
+
+```rust
+#[pyclass(from_py_object, frozen, name = "SessionContext", module = "datafusion", subclass)]
+#[derive(Clone)]
+pub struct PySessionContext {
+ pub ctx: SessionContext,
+}
+```
+
+Occasionally a type must remain mutable—for example when PyO3 attribute setters
+need to update fields directly. In these rare cases add an inline justification
+so reviewers and future contributors understand why `frozen` is unsafe to
+enable. `DataTypeMap` in {file}`src/common/data_type.rs` includes such a
+comment because PyO3 still needs to track field updates:
+
+```rust
+// TODO: This looks like this needs pyo3 tracking so leaving unfrozen for now
+#[derive(Debug, Clone)]
+#[pyclass(from_py_object, name = "DataTypeMap", module = "datafusion.common", subclass)]
+pub struct DataTypeMap {
+ #[pyo3(get, set)]
+ pub arrow_type: PyDataType,
+ #[pyo3(get, set)]
+ pub python_type: PythonType,
+ #[pyo3(get, set)]
+ pub sql_type: SqlType,
+}
+```
+
+When reviewers encounter a mutable `#[pyclass]` without a comment, they should
+request an explanation or ask that `frozen` be added. Keeping these wrappers
+frozen by default helps avoid subtle bugs stemming from PyO3's interior
+mutability tracking.
diff --git a/_sources/extension-guide/bundles.md.txt b/_sources/extension-guide/bundles.md.txt
new file mode 100644
index 00000000..437deb16
--- /dev/null
+++ b/_sources/extension-guide/bundles.md.txt
@@ -0,0 +1,313 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(extension_bundles)=
+
+# Extension bundles
+
+If your library ships codecs, or a query planner, or both, expose a **bundle**
+and let callers install it with
+{py:meth}`~datafusion.SessionContext.with_extensions`. This is the recommended
+way to package an extension, and the rest of this page explains what the
+bundle protocol asks of you and why.
+
+## Why not have callers install the pieces
+
+Installing the pieces by hand works, but it makes the caller responsible for
+ordering: the codecs have to be installed before the planner, because a planner
+is built against whatever codec chains exist when it is installed, and a codec
+added afterwards rebinds it. Get that wrong and the planner encodes through a
+chain that is missing a library.
+
+`with_extensions` removes the ordering question. An extension library exposes a
+bundle object implementing one or both of two hooks:
+
+```python
+class MyEngineExtension:
+ def __datafusion_session_components__(self, ctx: SessionContext) -> SessionExtensionComponents:
+ # Phase one. Create fresh components bound to `ctx` on every call.
+ return SessionExtensionComponents(
+ logical_extension_codecs=(self._make_logical_codec(ctx),),
+ physical_extension_codecs=(self._make_physical_codec(ctx),),
+ )
+
+ def __datafusion_session_planner__(self, ctx: SessionContext, fallback):
+ # Phase two. `ctx` now carries every bundle's codecs, and `fallback` is
+ # the planner built so far. Wrapping it is what makes this library
+ # compose with the other planners in the call.
+ return self._make_planner(ctx, fallback=fallback)
+```
+
+Implement whichever apply: a codec-only library defines the first, a library
+that ships only an optimizing planner defines the second. The caller then
+writes:
+
+```python
+ctx = SessionContext(config).with_extensions(lib_a.Extension(), lib_b.Extension())
+ctx.register_table("t", lib_a.TableProvider())
+ctx.register_udf(udf(lib_b.SomeUDF()))
+```
+
+`MyPlannerExtension` in [`datafusion-ffi-query-planner-example`] is a complete
+Rust implementation of the protocol, including taking the task-context provider
+off the supplied context, wrapping its codecs in `BundledLogicalCodec` /
+`BundledPhysicalCodec` so they carry declared ids, and constructing a Python
+{py:class}`~datafusion.SessionExtensionComponents`.
+
+## A bundle you can run
+
+Before wiring up a cdylib, it is worth seeing the protocol work end to end in
+pure Python. This runs against the plain wheel — the codec here re-exports the
+host session's own capsule, where a real one would return its library's:
+
+```python
+from datafusion import SessionContext, SessionExtensionComponents
+
+
+class Codec:
+ """Wraps a capsule so it carries an id. A Rust library ships this shape."""
+
+ def __init__(self, codec_id, capsule):
+ self.__datafusion_codec_id__ = codec_id
+ self._capsule = capsule
+
+ def __datafusion_logical_extension_codec__(self, session=None):
+ return self._capsule
+
+
+class Bundle:
+ def __init__(self, codec_id):
+ self.codec_id = codec_id
+
+ def __datafusion_session_components__(self, ctx):
+ # Fresh components on every call, bound to the `ctx` handed in.
+ # Never cache these, and never retain `ctx`.
+ return SessionExtensionComponents(
+ logical_extension_codecs=(
+ Codec(self.codec_id, ctx.__datafusion_logical_extension_codec__()),
+ )
+ )
+
+
+ctx = SessionContext().with_extensions(Bundle("tables.v1"), Bundle("engine.v1"))
+ctx.logical_extension_codec_ids()
+# ['tables.v1', 'engine.v1']
+```
+
+Three things this makes observable, each pinned by a test in
+`python/tests/test_context.py`:
+
+- **Ids accumulate in bundle order.** Decoding does not depend on that order;
+ only encoding does. See {ref}`extension_codec_order`.
+- **Two bundles claiming one id are refused**, with a `ValueError` naming the
+ id — not resolved by position, since a positional id would break stored plans
+ the first time a bundle reordered what it returns.
+- **A hook that raises leaves the receiving session untouched.** Add a bundle
+ whose hook raises and the source context's
+ {py:meth}`~datafusion.SessionContext.logical_extension_codec_ids` is still
+ empty afterwards.
+
+(extension_bundles_two_phases)=
+
+## Two phases, because codecs and planners compose differently
+
+A session chains **many** codecs and dispatches between them by id. Codecs
+therefore just accumulate: order affects encoding only, and decoding always
+routes to the codec that wrote the payload. A session holds exactly **one**
+query planner, so planners cannot accumulate — they compose by *nesting*, each
+wrapping the one before it and delegating to it for work it does not handle.
+
+So `with_extensions` runs every `__datafusion_session_components__` and installs
+all the codecs, and only then runs each `__datafusion_session_planner__`, in
+argument order, handing each the planner built so far. Two consequences worth
+holding onto:
+
+- **Bundle order matters differently for each.** For planners it sets the
+ nesting: the last extension listed ends up outermost and is consulted first.
+ For codecs it never affects decoding, and affects encoding only when two
+ codecs would claim the same node — see {ref}`extension_codec_order`.
+- **A planner is always built against the complete codec set**, including
+ codecs from bundles listed after it. This is what the low-level chaining
+ cannot give you, and it matters most for a nested planner: the rebuild that
+ follows a later codec install reaches only the outermost layer (see
+ {ref}`planner_codec_rebinding`), so a fallback captured before the codecs
+ were complete would stay stale forever.
+
+The two hooks therefore see the same session through different chains. Both
+receive a handle on the one session, so the task-context provider taken off
+either is the same and stays valid — but the `ctx` in phase one still carries
+the chains the receiver had, since nothing is installed yet, while the `ctx` in
+phase two carries every bundle's codecs. A bundle that reads the host's codec
+chains — `MyPlannerExtension` does, to give its planner the host's codecs
+rather than minting its own — must do that in the planner hook. Reading them in
+phase one gets the chains from before the call, missing even the bundle's own
+codecs.
+
+## Returning a planner, or not
+
+An extension that ignores `fallback` and returns an unrelated planner replaces
+every layer beneath it, including any planner the session already had. That is
+legal — a library that must be the only planner does it deliberately — but it
+is not composable, and nothing detects it. Returning `None` contributes no
+planner and leaves `fallback` in place.
+
+`None` is the no-op, not `fallback`. The capsule handed to the first bundle
+wraps the session's planner for export, so returning it unchanged installs that
+planner as a foreign one and every plan built afterwards crosses an FFI
+boundary it did not before. A bundle that decides at runtime it has nothing to
+contribute returns `None`.
+
+Three libraries that each ship a planner therefore install like this, with the
+outermost last:
+
+```python
+ctx = SessionContext(config).with_extensions(
+ tables.Extension(), # codecs only
+ functions.Extension(), # codecs only
+ optimizer.Extension(), # planner, wrapping the session default
+ distributed.Extension(), # planner, wrapping the optimizer
+)
+```
+
+(extension_bundles_order_conflict)=
+
+## When the two orders conflict
+
+Because codec position and planner position both come from one argument list, a
+library can in principle need to be early for one and late for the other: its
+codec must precede a broad claimer, while its planner must nest outside that
+library's planner.
+
+Do not try to satisfy both by reordering — contribute each half at its own
+position. The two hooks are independent, so a three-line adapter each is
+enough:
+
+```python
+class CodecsOf:
+ """Contribute only the codec half of a bundle, at this position."""
+ def __init__(self, inner):
+ self.inner = inner
+
+ def __datafusion_session_components__(self, ctx):
+ return self.inner.__datafusion_session_components__(ctx)
+
+
+class PlannerOf:
+ """Contribute only the planner half of a bundle, at this position."""
+ def __init__(self, inner):
+ self.inner = inner
+
+ def __datafusion_session_planner__(self, ctx, fallback):
+ return self.inner.__datafusion_session_planner__(ctx, fallback)
+
+
+ctx = SessionContext(config).with_extensions(
+ CodecsOf(engine), CodecsOf(tables), # engine's codec first
+ PlannerOf(tables), PlannerOf(engine), # engine's planner outermost
+)
+```
+
+This keeps everything `with_extensions` guarantees: one transaction, codecs
+complete before any planner is built, codec ids untouched — an id is read off
+the codec object, not off the extension that contributed it, so splitting a
+bundle cannot re-tag its payloads. A library that expects to be composed this
+way should expose the halves itself rather than make callers write the
+adapters.
+
+There is no attempt here to make every permutation expressible from one call.
+Two positions per bundle covers the cases that arise; anything stranger is a
+sign the libraries disagree about what they own, which is better fixed there.
+
+The low-level
+{py:meth}`~datafusion.SessionContext.with_logical_extension_codec` /
+{py:meth}`~datafusion.SessionContext.with_physical_extension_codec` /
+{py:meth}`~datafusion.SessionContext.set_query_planner` sequence also works,
+and it is the right answer when the pieces do not come as bundles at all. But
+it is a real downgrade, not just a more verbose spelling: you take back
+responsibility for installing every codec before every planner, and a planner
+you layer by hand keeps the codecs it captured — the
+{ref}`one-level rebind <planner_codec_rebinding>` does not reach inside it.
+Reach for it last.
+
+(extension_bundles_codecs_are_objects)=
+
+## Codecs are objects, not capsules
+
+`with_extensions` requires each codec to be an object exposing the capsule
+getter, and refuses a bare `PyCapsule`. A codec's id is read off the object it
+is handed over as, and a capsule has no type to read one from; since this
+method takes no `codec_id=`, there would be nothing left to name it by. A
+library holding a raw capsule — which is what a Rust implementation has —
+wraps it:
+
+```python
+class MyLogicalCodec:
+ # Optional. Without it the id is this class's import path, which is already
+ # stable; declare it if you may rename the class and need old plans to decode.
+ __datafusion_codec_id__ = "my_library.logical.v1"
+
+ def __init__(self, capsule):
+ self._capsule = capsule
+
+ def __datafusion_logical_extension_codec__(self, session=None):
+ return self._capsule
+```
+
+Wrapping is not just bookkeeping. It ties the id to the codec rather than to
+the bundle that contributed it, and that difference is load-bearing: an
+application commonly presents several libraries as one bundle of its own, and
+the id has to survive that. Were the id taken from the contributing bundle,
+wrapping `my_engine.Extension` inside `my_app.Extension` would silently re-tag
+the engine's payloads, and a scheduler that installs the engine's codec by its
+documented id would fail to decode plans from composed clients while succeeding
+for direct ones. The wrapper travels with the codec; the bundle does not.
+
+The query planner is exempt — it carries no wire id, so it may be an object or
+a capsule.
+
+(extension_bundles_transaction)=
+
+## Failure and rollback
+
+Nothing is written to the session until every factory has returned and every
+capsule has been validated, so a factory that raises leaves the session exactly
+as it was. A factory that mutates the context it is handed — registering a
+table, say — is **not** rolled back, which is why bundle objects must be
+configuration-only: create fresh components on each call, never cache bound
+components, and do not retain the context passed in.
+
+Declaring a component is what buys you that guarantee. Anything you return from
+your hook is validated while a failure still costs nothing, and is written only
+after every bundle in the call has succeeded. Anything you register yourself is
+written immediately, before the other bundles have even run. The ordering that
+makes this hold is recorded at {ref}`ffi_internals_commit_order`.
+
+Like every other derivation, the returned context is a handle on the *same*
+session as the receiver — see {ref}`extension_sessions`. Only the Python-side
+codec chains belong to the returned handle; the planner is installed on the
+shared session and takes effect even if that handle is discarded.
+
+`with_extensions` sidesteps the {ref}`one-level rebind <planner_codec_rebinding>`
+entirely, and for nested planners too: every codec from every bundle is
+installed before the first planner hook runs, so no layer — outer or fallback —
+is ever captured against a partial chain. There is no "afterwards" within a
+call. Prefer it over hand-layering whenever the planners you are composing all
+ship as bundles.
+
+[`datafusion-ffi-query-planner-example`]: https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-query-planner-example
diff --git a/_sources/extension-guide/capsule-protocol.md.txt b/_sources/extension-guide/capsule-protocol.md.txt
new file mode 100644
index 00000000..3b8331b4
--- /dev/null
+++ b/_sources/extension-guide/capsule-protocol.md.txt
@@ -0,0 +1,238 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(extension_capsule_protocol)=
+
+# The capsule protocol
+
+Every integration point in this section works the same way. Your library
+exposes a dunder method, datafusion-python calls it, and it hands back a
+`PyCapsule` wrapping an FFI-safe struct. This page describes that mechanism
+once; the pages after it describe what goes inside the capsule for each kind
+of component.
+
+The bulk of the code necessary to perform our FFI operations is in the
+upstream [DataFusion](https://datafusion.apache.org/) core repository. You can
+review the code and documentation in the [datafusion-ffi] crate.
+
+## FFI-safe types
+
+Our FFI implementation is narrowly focused on sharing data and functions with
+Rust backed libraries. This allows us to use the
+[stabby crate](https://crates.io/crates/stabby), which converts between Rust
+native types and FFI-safe alternatives. For example, if you needed to pass a
+`Vec<String>` via FFI, you can convert it to a
+`stabby::vec::Vec<stabby::string::String>` — the crate's own examples alias
+these as `SVec` and `SString`, which is the convention [datafusion-ffi] follows
+too.
+
+For `Option` and `Result`, [datafusion-ffi] defines its own `FFI_Option<T>` and
+`FFI_Result<T>` rather than using stabby's. Stabby's versions require
+`T: IStable` for niche optimization, and many of the `FFI_*` structs hold
+self-referential function pointers that cannot implement it.
+
+## `FFI_` on the provider, `Foreign` on the receiver
+
+The [datafusion-ffi] crate has been designed to make it easy to convert from
+DataFusion traits into their FFI counterparts. For example, if you have
+defined a custom
+[TableProvider](https://docs.rs/datafusion/45.0.0/datafusion/catalog/trait.TableProvider.html)
+and you want to create a sharable FFI counterpart, you could write:
+
+```rust
+let my_provider = Arc::new(MyTableProvider::default());
+let ffi_provider = FFI_TableProvider::new_with_ffi_codec(my_provider, false, None, codec);
+```
+
+where `codec` is the host's logical codec, read off the argument your getter
+was handed — see {ref}`extension_getter_argument`.
+
+If you were interfacing with a library that provided the above
+`FFI_TableProvider` and you needed a usable `TableProvider` back, you convert
+it into an `Arc<dyn TableProvider>`:
+
+```rust
+let provider: Arc<dyn TableProvider> = (&ffi_provider).into();
+```
+
+If you review the code in [datafusion-ffi] you will find that each of the
+traits we share across the boundary has two portions, one with an `FFI_`
+prefix and one with a `Foreign` prefix. This is used to distinguish which side
+of the FFI boundary that struct is designed to be used on. The structures with
+the `FFI_` prefix are to be used on the **provider** of the structure. In the
+example we're showing, this means the code that has written the underlying
+`TableProvider` implementation to access your custom data source. The
+structures with the `Foreign` prefix are to be used by the receiver. In this
+case, it is the `datafusion-python` library.
+
+Convert to the trait object rather than naming `ForeignTableProvider` yourself.
+The conversion compares the provider's library marker against the receiver's:
+when both sides turn out to be the same shared library it hands back the
+original `Arc` and skips the boundary entirely, and only otherwise wraps it in
+a `ForeignTableProvider`. Which one you get is an implementation detail, and
+both implement `TableProvider`.
+
+## Wrapping it in a capsule
+
+In order to share these FFI structures, we need to wrap them in some kind of
+Python object that can be used to interface from one package to another. As
+described in {ref}`extension_why_ffi`, we use `PyCapsule`. We can create a
+`PyCapsule` for our provider thusly:
+
+```rust
+PyCapsule::new_with_value(py, ffi_provider, cr"datafusion_table_provider")
+```
+
+On the receiving side, read the `FFI_TableProvider` back out of the capsule and
+convert it, which is what `table_provider_from_pycapsule` in `crates/util` does:
+
+```rust
+validate_pycapsule(capsule, "datafusion_table_provider")?;
+let data: NonNull<FFI_TableProvider> = capsule
+ .pointer_checked(Some(c"datafusion_table_provider"))?
+ .cast();
+let ffi_provider = unsafe { data.as_ref() };
+check_ffi_version("table provider", unsafe { (ffi_provider.version)() })?;
+let provider: Arc<dyn TableProvider> = ffi_provider.into();
+```
+
+## The naming rule
+
+The getter's name and the capsule's name are both fixed by the protocol, and
+they follow one rule with no exceptions:
+
+- The method is `__datafusion_<thing>__`.
+- The capsule it returns is named `datafusion_<thing>` — the same string
+ without the underscores.
+
+So a table provider is reached by calling `__datafusion_table_provider__` and
+must return a capsule named `datafusion_table_provider`. Return a capsule with
+the wrong name and the import fails with an error naming both the name found
+and the name expected, rather than reading the pointer as the wrong type.
+
+The full list of hooks is in the
+{ref}`hook reference <extension_guide>`. `TableProvider` was the first
+extension written this way and is the most thoroughly implemented; every hook
+added since follows the same pattern.
+
+## Version checking
+
+Objects imported through this protocol are checked against the major version of
+`datafusion-ffi` that datafusion-python was built with. A component produced by
+a library built against a different DataFusion major version raises an
+`ImportError` naming the version found and the version expected.
+
+This is a diagnostic rather than a soundness guarantee — reading the version
+out of the struct already assumes the local field layout — but it turns the
+common "extension library built against the wrong DataFusion" mistake into a
+clear message rather than undefined behaviour on first use. See
+{ref}`extension_version_mismatch`.
+
+Three FFI structs carry no version field and so cannot be checked:
+`FFI_TaskContextProvider`, `FFI_TableProviderFactory`, and
+`FFI_ExtensionOptions`.
+
+(extension_getter_argument)=
+
+## What your getter receives
+
+Most getters take one positional argument beyond `py`. The
+{ref}`hook reference <extension_guide>` says which, and the group that does
+looks like this:
+
+```rust
+fn __datafusion_physical_extension_codec__<'py>(
+ &self,
+ py: Python<'py>,
+ session: Bound<'py, PyAny>,
+) -> PyResult<Bound<'py, PyCapsule>> {
+ let runtime = get_tokio_runtime().handle().clone();
+ let ctx_provider = ffi_task_context_provider_from_pycapsule(&session)?;
+ let ffi = FFI_PhysicalExtensionCodec::new(inner, Some(runtime), ctx_provider);
+ PyCapsule::new_with_value(py, ffi, cr"datafusion_physical_extension_codec")
+}
+```
+
+This exists because the FFI constructors need things an extension library does
+not have. `FFI_{Logical,Physical}ExtensionCodec::new` needs a
+`TaskContextProvider` for the decode callbacks the codec will receive, and
+`FFI_QueryPlanner::new` needs both codecs on top of that. Taking them from the
+argument is what keeps a library from constructing a `SessionContext` purely to
+satisfy a parameter — an empty one resolves nothing, and
+`FFI_TaskContextProvider` holds it weakly, so a context built inline in the
+getter is already dropped by the time the capsule is used.
+
+### It is not always a session
+
+The parameter is conventionally named `session`, and for the codec and planner
+hooks it genuinely is one. For the provider and catalog hooks it may instead be
+a bare `datafusion_logical_extension_codec` capsule: `SessionContext.register_table`
+passes the session, while `Schema.register_table`,
+`SessionContext.register_catalog_provider`, `register_catalog_provider_list`,
+and `register_table_factory` pass the host's codec directly.
+
+This is why the helpers accept either. `ffi_logical_codec_from_pycapsule`
+calls the codec getter if the object has one and returns the object untouched
+if it does not, so the same line works in both cases:
+
+```rust
+let codec = ffi_logical_codec_from_pycapsule(session, None)?;
+```
+
+The rule to hold onto is therefore about capability rather than type: **the
+argument is something you can read the host's logical extension codec off.**
+The hooks that need more than a codec — the two extension codecs and the query
+planner, which need a task-context provider — are exactly the hooks that are
+always handed a real session.
+
+### Duck-type it
+
+Do not check the argument's type. Beyond the codec-capsule case above, even
+when it *is* a session it is the PyO3 context the binding installs through, not
+the `datafusion.context.SessionContext` wrapper. It carries every capsule
+getter and `__datafusion_codec_id__` — everything the protocol asks of it — but
+`isinstance(session, SessionContext)` is `False` in Python even though its
+`repr` reads `datafusion.SessionContext`.
+
+The two bundle hooks are the exception: `__datafusion_session_components__` and
+`__datafusion_session_planner__` are dispatched from Python by
+{py:meth}`~datafusion.SessionContext.with_extensions`, so they receive the
+wrapper. See {doc}`bundles`.
+
+`SessionContext` accepts the argument on its own codec and planner getters and
+ignores it, so a session satisfies the same protocol an extension library
+implements. When you export the current planner in order to wrap it,
+`ctx.__datafusion_query_planner__()` and `ctx.__datafusion_query_planner__(ctx)`
+are both fine.
+
+(extension_task_context_provider)=
+
+## `__datafusion_task_context_provider__`
+
+This is the one hook implemented by the **host** rather than by your library.
+You never define it; you read it off the session you were handed, which is what
+`ffi_task_context_provider_from_pycapsule` does. Nothing in datafusion-python
+calls it on a foreign object.
+
+Taking the host's provider means your decode callbacks resolve names against
+the session that is actually running the query — see
+{ref}`extension_codec_decode_session` — and it removes any need for your
+library to construct a `SessionContext` of its own.
+
+[datafusion-ffi]: https://crates.io/crates/datafusion-ffi
diff --git a/_sources/extension-guide/checklist.md.txt b/_sources/extension-guide/checklist.md.txt
new file mode 100644
index 00000000..6e204b3f
--- /dev/null
+++ b/_sources/extension-guide/checklist.md.txt
@@ -0,0 +1,99 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(extension_checklist)=
+
+# Extension author checklist
+
+The rules in this section, gathered into one list to run through before you
+publish. Each links to the page that explains it.
+
+## Protocol
+
+- [ ] **Every getter's capsule name matches its method name.**
+ `__datafusion_<thing>__` returns a capsule named `datafusion_<thing>`.
+ → {ref}`extension_capsule_protocol`
+- [ ] **Your getter does not inspect its argument.** Pass it to
+ `ffi_logical_codec_from_pycapsule` and move on. It is not always a
+ session, and when it is, it is not the Python `SessionContext` wrapper.
+ → {ref}`extension_getter_argument`
+- [ ] **You never construct a `SessionContext` inside your library.** Take
+ what the FFI constructors need off the argument you were handed. A
+ context built inline is already dropped by the time the capsule is used.
+ → {ref}`extension_getter_argument`
+- [ ] **You do not depend on the `datafusion-python` crate.**
+ → {ref}`extension_why_ffi`
+
+## Codecs
+
+- [ ] **Your codec claims narrowly.** Downcast to your own types. Claiming a
+ broad category takes nodes from every library installed after you and
+ makes your library order-sensitive for everyone downstream.
+ → {ref}`extension_codec_order`
+- [ ] **You declare `__datafusion_codec_id__` if you might rename the class.**
+ The default id is the exporting class's import path, so a rename stops
+ older plans decoding. → {ref}`extension_codec_ids`
+- [ ] **Name-only decoders check `name` before trusting `buf`.** An empty
+ payload has no id to route on, so your `try_decode_udf` can be called
+ with another library's function name and an empty buffer.
+ → {ref}`extension_codecs`
+- [ ] **You round-trip a plan in a test and assert *your* codec did the work.**
+ Both being installed does not mean your node reached you.
+ → {ref}`extension_codec_order`
+
+## Bundles and planners
+
+- [ ] **You ship a bundle, not loose pieces**, if you have codecs or a planner.
+ → {ref}`extension_bundles`
+- [ ] **Your bundle is configuration-only.** Fresh components on every call,
+ no cached bound components, no retaining the context passed in, no
+ registering anything on it — a factory that mutates the context is not
+ rolled back if a later factory raises.
+ → {ref}`extension_bundles`
+- [ ] **Your codecs are objects exposing the getter, not bare capsules.**
+ `with_extensions` refuses a capsule, because there would be nothing to
+ name the codec by. → {ref}`extension_bundles_codecs_are_objects`
+- [ ] **Your planner hook wraps `fallback` and delegates to it.** Ignoring it
+ replaces every layer beneath you, which is legal but not composable.
+ → {ref}`extension_bundles`
+- [ ] **Your planner hook returns `None`, not `fallback`, when it has nothing
+ to contribute.** Returning `fallback` installs the session's own planner
+ as a foreign one and adds an FFI hop that was not there.
+ → {ref}`extension_bundles`
+- [ ] **You read the host's codec chains in the planner hook, not the extension
+ hook.** Phase one runs before anything is installed.
+ → {ref}`extension_bundles_two_phases`
+- [ ] **If you also offer the low-level path**, document that codecs go in
+ before a layered planner. → {ref}`planner_codec_rebinding`
+
+## Packaging and documentation
+
+- [ ] **You state which `datafusion` version your release requires.** A
+ mismatch raises an `ImportError` on import, which is a good failure — but
+ only if your users know what to install. → {ref}`extension_version_mismatch`
+- [ ] **You tell your users to keep a context alive** for as long as anything
+ derived from it is in use. This is the rule most likely to arrive as a
+ bug report against your library. → {ref}`extension_sessions`
+- [ ] **Your production codec serializes durable metadata**, not a
+ process-local token. The examples in this repository use tokens to make
+ ownership observable; that is a demonstration, not a pattern.
+ → {ref}`extension_codec_durable_metadata`
+- [ ] **You have integration tests across a real FFI boundary.** The two
+ example crates in this repository are the pattern: build the cdylib,
+ install the wheel, then exercise it from Python.
diff --git a/_sources/extension-guide/codecs.md.txt b/_sources/extension-guide/codecs.md.txt
new file mode 100644
index 00000000..16043992
--- /dev/null
+++ b/_sources/extension-guide/codecs.md.txt
@@ -0,0 +1,183 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(extension_codecs)=
+
+# Extension codecs
+
+A codec is what lets your objects survive being serialized into a plan and
+rebuilt somewhere else — another process, or another program. If your library
+contributes table providers, functions, or execution plan nodes and those plans
+have to leave the process, you need one.
+
+Codecs are contributed to a session either through an
+{ref}`extension bundle <extension_bundles>`, which is the recommended route,
+or one at a time through
+{py:meth}`~datafusion.SessionContext.with_logical_extension_codec` and
+{py:meth}`~datafusion.SessionContext.with_physical_extension_codec`.
+
+## Codecs compose
+
+Each call to `with_logical_extension_codec` or
+`with_physical_extension_codec` **appends** the codec to the session's codec
+chain rather than replacing prior codecs. One session can therefore carry
+codecs from several independent libraries at once.
+
+**Nothing is asked of the codec itself.** Implement `LogicalExtensionCodec` or
+`PhysicalExtensionCodec` exactly as you would for a session that installs only
+yours. When your codec writes bytes into a serialized plan, datafusion-python
+records which codec wrote them, and strips that record off again before handing
+the bytes back. So your codec receives, byte for byte, the payload it wrote,
+and is never offered a payload another codec wrote.
+
+A codec that also ships to hosts which dispatch differently may still want its
+own guard against foreign payloads. Keeping one is fine; it is simply not
+needed for the datafusion-python path.
+
+(extension_codec_durable_metadata)=
+
+## Encode metadata, not a handle to a live object
+
+Your payload has to be enough to rebuild the object somewhere your process is
+not. Write the metadata a fresh instance can be constructed from — a path, a
+connection string, a schema, the options the object was created with.
+
+The example codecs in this repository do not do this, and it is worth knowing
+before copying them. They keep a process-local `HashMap` of live providers and
+encode an integer token into it: encoding inserts, decoding removes. That makes
+Rust type identity observable across three separately loaded libraries in one
+test, which is what the examples exist to show. It also means a decode consumes
+its token, so the same bytes cannot be decoded twice, one encoded plan cannot
+fan out to several readers, and a plan that never reaches a decoder keeps its
+provider alive for the life of the process. A real codec has none of those
+properties because it does not park the object anywhere.
+
+(extension_codec_ids)=
+
+## Codec ids
+
+That record is the codec's **id**: a short string stored inside the plan,
+naming the codec that wrote each payload. Because plans are decoded in another
+process — or another program — the id has to name the same codec there as it
+did where the plan was written.
+
+Ids are assigned for you. A codec's id is normally its exporting class's import
+path, such as `my_library.Codec`, which is what you will see in
+{py:meth}`~datafusion.SessionContext.logical_extension_codec_ids` and in decode
+errors. You choose one yourself in three cases:
+
+- **Two instances of one class.** Both get the same id, so the second install
+ raises `ValueError`. Pass `codec_id=` to tell them apart.
+- **A bare `PyCapsule`.** A capsule has no class to take a name from, so
+ installing one through `with_logical_extension_codec` or
+ `with_physical_extension_codec` gives it an id private to the session that
+ installed it; plans it encodes fail with a clear error on any other session
+ rather than being decoded by the wrong codec. Pass `codec_id=` if those plans
+ have to cross sessions.
+
+ {py:meth}`~datafusion.SessionContext.with_extensions` takes no `codec_id=`,
+ so it refuses a bare capsule outright and tells you to wrap it. See
+ {ref}`extension_bundles_codecs_are_objects`.
+- **A class you intend to rename.** The id follows the class name, so renaming
+ stops older plans from decoding. Declare `__datafusion_codec_id__` on the
+ exporting object to pin an id that survives the rename.
+
+{py:meth}`~datafusion.SessionContext.logical_extension_codec_ids` and its
+physical counterpart list the ids installed on a session, which is also what a
+decode failure names.
+
+Installing one context's codec stack on another session composes the two
+sessions rather than copying codecs out of one: the imported codecs resolve
+their task context against the original and stop working when it is dropped —
+see {ref}`ffi_internals_one_arc`. Pass the context itself rather than the
+capsule it exports, so its codecs get an id that other sessions can decode.
+
+## Functions whose name is the whole encoding
+
+A codec may own functions that need no payload at all, where the name is the
+whole encoding: `try_encode_udf` writes nothing and `try_decode_udf` rebuilds
+the function from `name`. That is supported and needs no id, because an `Ok`
+with an empty buffer is read as "no opinion" and passes the object to the next
+codec. `NameOnlyUdfCodec` in [`datafusion-ffi-example`] is the worked case.
+Anything no installed codec claims falls through to
+`Default{Logical,Physical}ExtensionCodec`.
+
+This is the one case where your decoder is consulted about something you may
+not own, because an empty payload has no id to route on. `try_decode_udf` and
+its aggregate and window siblings can therefore be called with an empty `buf`
+and a `name` belonging to another library. Decide from `name` and return an
+error if it is not yours; do not assume `buf` is non-empty.
+
+:::{note}
+The framing itself — how an id is stored alongside a payload and routed back,
+and the two cases that stay unframed — is internal to datafusion-python and
+documented in `crates/core/src/codec.rs` for anyone changing it.
+:::
+
+The current FFI logical codec supports providers and UDFs but not arbitrary
+custom `LogicalPlan::Extension` nodes. See both example READMEs for the
+supported flow and local build commands.
+
+(extension_codec_order)=
+
+## When codec order matters
+
+Decoding is never order-dependent: a payload names its codec by id and the
+chain dispatches straight to it. Encoding walks the chain in install order and
+stops at the first codec that claims the node. Most of the time that is
+invisible, because libraries claim disjoint things — one owns its table
+providers, another its UDFs, a third its own execution plan nodes.
+
+It stops being invisible when a codec claims *broadly*. A node that came from
+another library arrives as an opaque `ForeignExecutionPlan`, and a codec that
+claims any of those will take nodes it does not own from any library installed
+after it. The query still succeeds. What changes is which library wrote the
+bytes — so a plan that has to decode in another process now needs whichever
+library happened to win, not the one whose node it is.
+`MyPhysicalExtensionCodec` in [`datafusion-ffi-example`] claims this way, and
+the query-planner example's test suite pins the consequence.
+
+Two rules of thumb:
+
+- **Writing a codec, claim narrowly.** Downcast to your own types. Claiming a
+ broad category makes your library order-sensitive for everyone downstream of
+ it.
+- **Shipping plans out of the process, verify.** Do not assume your node
+ reached your codec just because both are installed. Round-trip a plan through
+ {py:meth}`ExecutionPlan.to_bytes <datafusion.ExecutionPlan.to_bytes>` /
+ {py:meth}`~datafusion.ExecutionPlan.from_bytes` in a test and assert your
+ codec did the work.
+
+(extension_codec_decode_session)=
+
+## A codec decodes against the session running the query
+
+Because the task-context provider comes from the host — see
+{ref}`extension_getter_argument` — a decode callback running inside an
+extension library resolves names against the session running the query. A
+function registered with `ctx.register_udf(...)` is visible to a foreign codec
+decoding a node that references it by name, and the handle is live rather than
+a snapshot, so a registration made after the codec is installed is visible too.
+
+This is covered in
+`examples/datafusion-ffi-query-planner-example/python/tests/_test_three_library_query_planner.py`,
+where the example codecs take a `require_udf_on_decode` name and resolve it out
+of the task context they are handed.
+
+[`datafusion-ffi-example`]: https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-example
diff --git a/_sources/extension-guide/functions.md.txt b/_sources/extension-guide/functions.md.txt
new file mode 100644
index 00000000..fcfef9f1
--- /dev/null
+++ b/_sources/extension-guide/functions.md.txt
@@ -0,0 +1,106 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(extension_functions)=
+
+# Functions and table functions
+
+Four hooks contribute functions written in Rust. Users can also define
+functions in pure Python — see
+{doc}`../user-guide/common-operations/udf-and-udfa` — and the two roads meet at
+the same registration methods.
+
+| Hook | Contributes | Wrapped by | Registered with |
+| --- | --- | --- | --- |
+| `__datafusion_scalar_udf__` | scalar function | {py:func}`datafusion.udf` | {py:meth}`~datafusion.SessionContext.register_udf` |
+| `__datafusion_aggregate_udf__` | aggregate function | {py:func}`datafusion.udaf` | {py:meth}`~datafusion.SessionContext.register_udaf` |
+| `__datafusion_window_udf__` | window function | {py:func}`datafusion.udwf` | {py:meth}`~datafusion.SessionContext.register_udwf` |
+| `__datafusion_table_function__` | function returning a table | {py:func}`datafusion.udtf` | {py:meth}`~datafusion.SessionContext.register_udtf` |
+
+All four are implemented in [`datafusion-ffi-example`], one per file.
+
+## The three scalar-shaped hooks
+
+The scalar, aggregate, and window getters take **no argument** beyond `py` —
+they need no codec and no task-context provider, because a function is
+identified by name and signature rather than by anything session-scoped:
+
+```rust
+#[pymethods]
+impl MyScalarUDF {
+ fn __datafusion_scalar_udf__<'py>(
+ &self,
+ py: Python<'py>,
+ ) -> PyResult<Bound<'py, PyCapsule>> {
+ let udf = Arc::new(ScalarUDF::from(self.clone()));
+ let ffi = FFI_ScalarUDF::from(udf);
+
+ PyCapsule::new_with_value(py, ffi, cr"datafusion_scalar_udf")
+ }
+}
+```
+
+Aggregate and window follow identically with `FFI_AggregateUDF` /
+`FFI_WindowUDF` and the matching capsule names.
+
+Your users wrap the object once and register the result:
+
+```python
+from datafusion import udf
+
+ctx.register_udf(udf(my_library.MyScalarUDF()))
+```
+
+## Table functions
+
+A table function takes literal `Expr` arguments and returns a table provider,
+so it needs the host's logical codec the way a
+{ref}`table provider <extension_providers>` does:
+
+```rust
+fn __datafusion_table_function__<'py>(
+ &self,
+ py: Python<'py>,
+ session: Bound<'py, PyAny>,
+) -> PyResult<Bound<'py, PyCapsule>> {
+ let func = self.clone();
+ let codec = ffi_logical_codec_from_pycapsule(session, None)?;
+ let provider = FFI_TableFunction::new_with_ffi_codec(Arc::new(func), None, codec);
+
+ PyCapsule::new_with_value(py, provider, cr"datafusion_table_function")
+}
+```
+
+Only literal expressions are supported as arguments. The Python side is
+described under
+{doc}`Table Functions <../user-guide/common-operations/udf-and-udfa>`.
+
+## Serializing functions
+
+A function that appears in a plan leaving the process has to be reconstructible
+on the far side. Functions are the one case where a codec often needs **no
+payload at all**: the name is the whole encoding, `try_encode_udf` writes
+nothing, and `try_decode_udf` rebuilds the function from `name`. See
+{ref}`extension_codecs` for how that works and for the one obligation it puts
+on your decoder — with an empty payload there is no id to route on, so your
+`try_decode_udf` can be called with a `name` belonging to another library.
+
+`NameOnlyUdfCodec` in [`datafusion-ffi-example`] is the worked case.
+
+[`datafusion-ffi-example`]: https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-example
diff --git a/_sources/extension-guide/index.md.txt b/_sources/extension-guide/index.md.txt
new file mode 100644
index 00000000..91d42a3c
--- /dev/null
+++ b/_sources/extension-guide/index.md.txt
@@ -0,0 +1,136 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(ffi)=
+(extension_guide)=
+
+# Extension Guide
+
+This section is for people **writing** a library that plugs into
+datafusion-python — a package that contributes table providers, functions, a
+catalog, extension codecs, or a query planner, usually written in Rust and
+exposed through [PyO3](https://pyo3.rs).
+
+Two neighbouring audiences are served elsewhere:
+
+- **Using** an extension library someone else published:
+ {ref}`user_guide_extensions`.
+- **Changing datafusion-python itself**, including the framing that makes this
+ protocol work: {doc}`../contributor-guide/index`.
+
+The protocol described here is a public, versioned contract. When a hook's
+signature changes, the change is documented in
+{doc}`../user-guide/upgrade-guides` with a before-and-after, and objects built
+against a different DataFusion major version are rejected with a clear error
+rather than used as-is.
+
+## Start here
+
+If you have not built an extension before, read {doc}`why-ffi` and
+{doc}`capsule-protocol` in order. Together they explain why your library must
+not depend on the `datafusion-python` crate, and the one convention every hook
+follows. After that the pages are independent — go to the one matching what
+you are contributing.
+
+## The three roles in a query
+
+A single query can involve three independent native libraries, and much of
+this section only makes sense once they are distinct in your head:
+
+- **datafusion-python** — the host. Owns the session, and decodes whatever
+ comes back from the other two.
+- **A provider library** — owns table providers, catalogs, and functions, plus
+ the codecs that serialize them.
+- **A planner library** — owns a query planner and the configuration it needs.
+
+The worked examples in this repository use two separate crates,
+[`datafusion-ffi-example`] and [`datafusion-ffi-query-planner-example`], so
+each role has a distinct shared-library identity. A real library may play more
+than one role; keeping them separate in the examples is what makes the
+boundaries observable.
+
+The session owns the codecs used for the exchange and supplies them to the
+foreign planner. That is what lets the planner decode provider-owned objects,
+and lets datafusion-python decode the physical plan the planner returns.
+
+## Hook reference
+
+Every integration point is a dunder method named `__datafusion_*__`. The
+convention is uniform enough to be worth stating once: your object exposes the
+getter, datafusion-python calls it, and it returns a `PyCapsule` wrapping an
+FFI-safe struct. See {doc}`capsule-protocol` for what that means and
+{ref}`extension_getter_argument` for the argument every getter in the middle
+group receives.
+
+Each hook's capsule name follows from its own name by
+{ref}`the naming rule <extension_capsule_protocol>`, so it is not repeated
+here.
+
+| Hook | Contributes | Argument | Documented on |
+| --- | --- | --- | --- |
+| `__datafusion_table_provider__` | one table | codec source | {doc}`table-providers` |
+| `__datafusion_table_provider_factory__` | tables built by `CREATE EXTERNAL TABLE` | codec source | {doc}`table-providers` |
+| `__datafusion_catalog_provider__` | a named set of schemas | codec source | {doc}`table-providers` |
+| `__datafusion_catalog_provider_list__` | the whole catalog namespace | codec source | {doc}`table-providers` |
+| `__datafusion_schema_provider__` | a named set of tables | codec source | {doc}`table-providers` |
+| `__datafusion_table_function__` | a table-valued function | session | {doc}`functions` |
+| `__datafusion_scalar_udf__` | a scalar function | none | {doc}`functions` |
+| `__datafusion_aggregate_udf__` | an aggregate function | none | {doc}`functions` |
+| `__datafusion_window_udf__` | a window function | none | {doc}`functions` |
+| `__datafusion_logical_extension_codec__` | a logical codec | session | {doc}`codecs` |
+| `__datafusion_physical_extension_codec__` | a physical codec | session | {doc}`codecs` |
+| `__datafusion_codec_id__` | the wire id a codec's payloads carry | — | {ref}`extension_codec_ids` |
+| `__datafusion_query_planner__` | a query planner | session | {doc}`query-planners` |
+| `__datafusion_session_components__` | a bundle's codecs | `ctx` | {doc}`bundles` |
+| `__datafusion_session_planner__` | a bundle's planner, wrapping `fallback` | `ctx`, `fallback` | {doc}`bundles` |
+| `__datafusion_physical_optimizer_rule__` | a physical optimizer rule | none | {ref}`extension_other_hooks` |
+| `__datafusion_extension_options__` | typed entries in the session config | none | {ref}`extension_other_hooks` |
+| `__datafusion_task_context_provider__` | the host's task context | none | {ref}`extension_task_context_provider` |
+
+Three rows are not like the others. `__datafusion_task_context_provider__` is
+implemented by the **host**, not by your library — you read it off the session
+you are handed. `__datafusion_codec_id__` is a plain string attribute rather
+than a method returning a capsule. And the two `session_` hooks are dispatched
+from Python and return objects rather than capsules.
+
+"codec source" in the argument column means the value is something you can
+read the host's logical extension codec off, which is not always a session.
+{ref}`extension_getter_argument` explains why, and what to do with it.
+
+`python/tests/test_docstrings.py` compares this table against the hook names
+the package actually dispatches, so a hook added or renamed without touching
+this page fails the suite.
+
+```{toctree}
+:maxdepth: 2
+
+why-ffi
+capsule-protocol
+table-providers
+functions
+codecs
+bundles
+query-planners
+other-components
+sessions
+checklist
+```
+
+[`datafusion-ffi-example`]: https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-example
+[`datafusion-ffi-query-planner-example`]: https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-query-planner-example
diff --git a/_sources/extension-guide/other-components.md.txt b/_sources/extension-guide/other-components.md.txt
new file mode 100644
index 00000000..919b8718
--- /dev/null
+++ b/_sources/extension-guide/other-components.md.txt
@@ -0,0 +1,72 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(extension_other_hooks)=
+
+# Optimizer rules and configuration
+
+Two hooks contribute things that are neither data nor functions: a rewrite pass
+over physical plans, and typed entries in the session config. Both take no
+argument and both are implemented in [`datafusion-ffi-example`].
+
+## A physical optimizer rule
+
+**`__datafusion_physical_optimizer_rule__`** contributes a rule that rewrites
+physical plans, installed with
+{py:meth}`~datafusion.SessionContext.add_physical_optimizer_rule`. Reach for
+this rather than a {doc}`query planner <query-planners>` when you want to
+adjust the plan DataFusion produced rather than produce it yourself — it is
+much the smaller commitment, and rules accumulate where planners nest.
+
+```rust
+fn __datafusion_physical_optimizer_rule__<'py>(
+ &self,
+ py: Python<'py>,
+) -> PyResult<Bound<'py, PyCapsule>> {
+ let rule: Arc<dyn PhysicalOptimizerRule + Send + Sync> = Arc::new(self.clone());
+ let runtime = get_tokio_runtime().handle().clone();
+ let ffi = FFI_PhysicalOptimizerRule::new(rule, Some(runtime));
+
+ PyCapsule::new_with_value(py, ffi, cr"datafusion_physical_optimizer_rule")
+}
+```
+
+## Typed configuration
+
+**`__datafusion_extension_options__`** contributes typed configuration entries
+that your components can read back out of the session config, installed with
+{py:meth}`SessionConfig.with_extension <datafusion.SessionConfig.with_extension>`.
+`FFI_ExtensionOptions` carries no version field, so it is one of the three
+components that cannot be version-checked on import.
+
+```rust
+fn __datafusion_extension_options__<'py>(
+ &self,
+ py: Python<'py>,
+) -> PyResult<Bound<'py, PyCapsule>> {
+ let mut config = FFI_ExtensionOptions::default();
+ config
+ .add_config(self)
+ .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+
+ PyCapsule::new_with_value(py, config, cr"datafusion_extension_options")
+}
+```
+
+[`datafusion-ffi-example`]: https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-example
diff --git a/_sources/extension-guide/query-planners.md.txt b/_sources/extension-guide/query-planners.md.txt
new file mode 100644
index 00000000..e373bb55
--- /dev/null
+++ b/_sources/extension-guide/query-planners.md.txt
@@ -0,0 +1,104 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(extension_planners)=
+
+# Query planners
+
+A query planner turns a logical plan into a physical one. Contributing your own
+is how a library changes the way queries execute rather than what data they can
+reach — a distributed engine is the motivating case, and an optimizing rewriter
+is another.
+
+The session owns the codecs used for the exchange and supplies them to your
+planner. That is what lets the planner decode provider-owned objects, and lets
+datafusion-python decode the physical plan the planner returns. Your planner
+uses `FFI_QueryPlanner::new_with_ffi_codecs` with the two codecs it takes off
+the session, and never touches a task-context provider directly. That also
+matches what installation does anyway:
+{py:meth}`~datafusion.SessionContext.set_query_planner` builds the planner
+against the codecs of the session that will run the query.
+
+`MyQueryPlanner` in [`datafusion-ffi-query-planner-example`] is the worked
+implementation.
+
+## One planner per session
+
+A session holds exactly one query planner. Calling `set_query_planner` again
+**replaces** the installed planner instead of layering another one.
+
+To chain planners, have the new planner wrap the capsule returned by
+{py:meth}`SessionContext.__datafusion_query_planner__ <datafusion.SessionContext.__datafusion_query_planner__>`,
+captured before the new planner is installed, and delegate to it explicitly:
+
+```python
+fallback = ctx.__datafusion_query_planner__()
+ctx.set_query_planner(MyPlanner(fallback=fallback))
+```
+
+`set_query_planner` returns nothing. The query planner lives in `SessionState`,
+so it is a property of the session rather than of a handle on it, and
+installing one is visible to every context sharing that session. See
+{ref}`extension_sessions`.
+
+If the planners you are composing all ship as
+{ref}`bundles <extension_bundles>`, prefer `with_extensions` — it does the
+nesting for you and cannot capture a partial codec chain.
+
+(planner_codec_rebinding)=
+
+## Install codecs before a layered planner
+
+Installing a codec on a session that already has a foreign planner rebuilds
+that planner against the new chain: there is one planner, and it has to carry
+the codecs currently in force. But that rebuild swaps the codecs on the
+installed `ForeignQueryPlanner` handle, and **only that handle**.
+
+A planner that wraps a fallback resolved that fallback when *it* was installed,
+and holds the result inside its own library's private data — behind a
+`create_physical_plan` function pointer, with no Python-side handle. A codec
+installed afterwards therefore reaches the outer planner and not the fallback,
+which keeps whichever codecs were in force when it was imported.
+
+The stale codecs stay usable rather than dangling — they hold weak handles to
+the one `Arc<SessionContext>` the session keeps alive — so the effect is a
+fallback hop serializing with an older codec, not a failure. Neither side can
+repair it; the reasons are in {ref}`ffi_internals_rebinding`, and a fix has to
+come from upstream
+([apache/datafusion#24762](https://github.com/apache/datafusion/issues/24762)).
+
+So, three rules:
+
+- **Install the codecs before a layered planner.**
+- If a codec has to go in afterwards, install the outer planner again *on the
+ handle that holds the new codec* — that re-runs its getter, which re-imports
+ the fallback against that handle's codecs. Re-installing on the original
+ handle rebinds the session's planner back to the original handle's codecs
+ instead, which is the trap the query-planner example's test suite pins.
+- Better, use {ref}`extension_bundles`, where there is no "afterwards" within a
+ call.
+
+:::{note}
+This is invisible in the examples here, which use one fallback in the same
+cdylib as its wrapper; `datafusion-ffi` short-circuits a same-library hop
+rather than serializing, so no codec runs. A fallback in a *different* library
+would serialize, and would do it with the codecs it was imported with.
+:::
+
+[`datafusion-ffi-query-planner-example`]: https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-query-planner-example
diff --git a/_sources/extension-guide/sessions.md.txt b/_sources/extension-guide/sessions.md.txt
new file mode 100644
index 00000000..21bfb6e3
--- /dev/null
+++ b/_sources/extension-guide/sessions.md.txt
@@ -0,0 +1,117 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(extension_sessions)=
+
+# Sessions, handles, and lifetimes
+
+One `SessionContext` in Python is a *handle* on a session, not the session
+itself. Several handles can share one session, and which handle you call
+something on sometimes matters. This page is the set of rules that follow from
+that — the ones an extension author trips over.
+
+## The rule that surprises people
+
+> The session's query planner carries the codecs of the handle that most
+> recently installed one. Every other path — `Expr.to_bytes(ctx)`,
+> `ExecutionPlan.to_bytes(ctx)`, registering a provider — uses the codecs of
+> the handle you call it on.
+
+Those can be different handles, and then one session has two codec chains in
+effect at once:
+
+```python
+ctx = ctx.with_logical_extension_codec(codec_a)
+ctx.set_query_planner(planner)
+ctx.with_logical_extension_codec(codec_b) # discarded
+
+Expr.to_bytes(expr, ctx) # encodes with [codec_a, default] -- ctx's own field
+ctx.sql(...).collect() # plans with [codec_b, codec_a, default] -- the discarded
+ # handle's chain, installed on the shared session
+```
+
+Chaining `ctx = ctx.with_...(...)` keeps the two in step, which is why every
+example in this guide does. The query-planner example's test suite pins the
+divergence.
+
+## What a derived context shares
+
+{py:meth}`~datafusion.SessionContext.with_logical_extension_codec`,
+{py:meth}`~datafusion.SessionContext.with_physical_extension_codec`,
+{py:meth}`~datafusion.SessionContext.with_python_udf_inlining`, and
+{py:meth}`~datafusion.SessionContext.with_extensions` return a new
+`SessionContext` wrapping the *same* underlying session. Only the Python-side
+codec settings differ; catalogs, tables, registered functions, and
+configuration are the one shared session, so a registration on either side is
+visible to both.
+
+There is one `Arc<SessionContext>` per session, which is what makes the weak
+task-context-provider scheme work: a component bound through any handle stays
+valid while *any* handle on that session is alive, so there is no way to bind a
+component to an intermediate handle and have it dangle when that handle is
+dropped. See {ref}`ffi_internals_one_arc` for why the allocation is kept
+rather than replaced.
+
+`set_query_planner` does not return anything, because the query planner lives
+in `SessionState` and is therefore a property of the session rather than of a
+handle on it. Installing one is visible to every context sharing that session —
+including ones a `with_*` call returned earlier. Installing a codec on a
+session that already has a foreign planner rebuilds that planner against the
+new chain for the same reason. This happens on the shared session, so it takes
+effect even if the returned context is discarded:
+`ctx.with_python_udf_inlining(...)` whose result is thrown away still leaves
+the session's planner carrying the codecs of that discarded handle. A call that
+changes nothing is exempt — asking for the inlining setting a context already
+has returns a handle without touching the session.
+
+Order between installing codecs and installing a planner is a readability
+preference rather than a requirement, since installing a codec after a planner
+rebuilds the planner against it. The exception is a *layered* planner, where
+codecs-first is a requirement: see {ref}`planner_codec_rebinding`.
+
+## Keep a context alive
+
+The session owns every installed component's task-context provider, and
+dependent objects do not extend its lifetime. A `DataFrame`, logical plan, or
+capsule can outlive every context on the session, but any operation that
+reaches an FFI codec after the last one is collected fails with:
+
+```text
+TaskContextProvider went out of scope over FFI boundary
+```
+
+Keep a context alive for as long as objects derived from it are in use. This is
+the rule most likely to reach your users as a bug report against your library,
+so it is worth stating in your own documentation too — the user-facing version
+is in {ref}`user_guide_extensions`.
+
+The same rule applies to a capsule you take off a context inside your own code:
+a codec capsule taken from a throwaway `SessionContext()` names a session that
+is already gone and fails on first use.
+
+:::{warning}
+{py:meth}`~datafusion.SessionContext.enable_url_table` is an exception to the
+one-session-one-allocation rule above: it clones the underlying
+`SessionContext`, so the returned context has an allocation of its own and must
+not outlive the receiver. It also forks the session's state while keeping its
+id, so two handles report one `session_id()` with divergent configuration. That
+is a bug rather than a design, tracked in
+[apache/datafusion-python#1708](https://github.com/apache/datafusion-python/issues/1708);
+do not build on the behaviour.
+:::
diff --git a/_sources/extension-guide/table-providers.md.txt b/_sources/extension-guide/table-providers.md.txt
new file mode 100644
index 00000000..8f5a0ee0
--- /dev/null
+++ b/_sources/extension-guide/table-providers.md.txt
@@ -0,0 +1,143 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(extension_providers)=
+
+# Providers and catalogs
+
+Five hooks expose data to a session, at four levels of granularity. All five
+follow the {ref}`capsule protocol <extension_capsule_protocol>` and all five
+receive a {ref}`codec source <extension_getter_argument>` as their single
+argument. Every one of them is implemented in [`datafusion-ffi-example`].
+
+| Hook | Exposes | Registered with |
+| --- | --- | --- |
+| `__datafusion_table_provider__` | one table | {py:meth}`SessionContext.register_table <datafusion.SessionContext.register_table>` |
+| `__datafusion_table_provider_factory__` | a factory that builds tables from `CREATE EXTERNAL TABLE` | {py:meth}`SessionContext.register_table_factory <datafusion.SessionContext.register_table_factory>` |
+| `__datafusion_schema_provider__` | a named set of tables | {py:meth}`Catalog.register_schema <datafusion.catalog.Catalog.register_schema>` |
+| `__datafusion_catalog_provider__` | a named set of schemas | {py:meth}`SessionContext.register_catalog_provider <datafusion.SessionContext.register_catalog_provider>` |
+| `__datafusion_catalog_provider_list__` | the whole catalog namespace | {py:meth}`SessionContext.register_catalog_provider_list <datafusion.SessionContext.register_catalog_provider_list>` |
+
+A schema provider is the one that does not register on the session: you reach a
+{py:class}`~datafusion.catalog.Catalog` first, with
+{py:meth}`SessionContext.catalog <datafusion.SessionContext.catalog>`, and
+register the schema on that.
+
+Start with a table provider. Reach for the schema and catalog levels when your
+data source has its own namespace that should be browsable rather than
+registered table by table, and for the provider list only when your library is
+replacing the catalog namespace outright.
+
+## A table provider
+
+Implement
+[TableProvider](https://datafusion.apache.org/library-user-guide/custom-table-providers.html)
+in Rust, then expose it:
+
+```rust
+#[pymethods]
+impl MyTableProvider {
+ fn __datafusion_table_provider__<'py>(
+ &self,
+ py: Python<'py>,
+ session: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyCapsule>> {
+ let provider = Arc::new(self.clone());
+ let codec = ffi_logical_codec_from_pycapsule(session, None)?;
+ let provider = FFI_TableProvider::new_with_ffi_codec(provider, false, None, codec);
+
+ PyCapsule::new_with_value(py, provider, cr"datafusion_table_provider")
+ }
+}
+```
+
+Your users then register it as they would any other table — see
+{ref}`io_custom_table_provider` for the Python side.
+
+## The catalog family
+
+The three catalog-level hooks have the same shape as each other. Taking the
+schema provider as the representative:
+
+```rust
+#[pymethods]
+impl MySchemaProvider {
+ fn __datafusion_schema_provider__<'py>(
+ &self,
+ py: Python<'py>,
+ codec: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyCapsule>> {
+ let provider = Arc::clone(&self.inner) as Arc<dyn SchemaProvider + Send>;
+
+ let codec = ffi_logical_codec_from_pycapsule(codec, None)?;
+ let provider = FFI_SchemaProvider::new_with_ffi_codec(provider, None, codec);
+
+ PyCapsule::new_with_value(py, provider, cr"datafusion_schema_provider")
+ }
+}
+```
+
+Swap `Schema` for `Catalog` or `CatalogProviderList` and the getter, capsule
+name, and FFI type change together, following {ref}`the naming rule
+<extension_capsule_protocol>`. `catalog_provider.rs` in
+[`datafusion-ffi-example`] implements all three in one file, which is the
+easiest way to see the symmetry.
+
+Note the parameter name. These four hooks are handed the host's logical codec
+directly rather than a session, so naming it `codec` is more honest than
+`session` — but do not rely on either: pass it to
+`ffi_logical_codec_from_pycapsule` and do not inspect it. See
+{ref}`extension_getter_argument`.
+
+## A table provider factory
+
+A factory backs `CREATE EXTERNAL TABLE`: DataFusion hands it the statement's
+options and it produces a provider. The getter takes the codec and wraps an
+`Arc<dyn TableProviderFactory>`:
+
+```rust
+fn __datafusion_table_provider_factory__<'py>(
+ &self,
+ py: Python<'py>,
+ codec: Bound<'py, PyAny>,
+) -> PyResult<Bound<'py, PyCapsule>> {
+ let codec = ffi_logical_codec_from_pycapsule(codec, None)?;
+ let factory = Arc::clone(&self.inner) as Arc<dyn TableProviderFactory + Send>;
+ let factory = FFI_TableProviderFactory::new_with_ffi_codec(factory, None, codec);
+
+ PyCapsule::new_with_value(py, factory, cr"datafusion_table_provider_factory")
+}
+```
+
+`FFI_TableProviderFactory` carries no version field, so a factory is one of the
+three components that cannot be version-checked on import. Be correspondingly
+careful about which DataFusion version you build against.
+
+## Serializing what you expose
+
+If plans referencing your tables have to leave the process — a distributed
+engine will make them — your library also needs a logical extension codec, so
+the provider can be rebuilt on the other side. That is {doc}`codecs`, and
+{doc}`bundles` is how you ship the two together.
+
+The codec you pass to `new_with_ffi_codec` above is the **host's**, used to
+serialize the parts of a plan the host owns. It is not a substitute for
+contributing your own.
+
+[`datafusion-ffi-example`]: https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-example
diff --git a/_sources/extension-guide/why-ffi.md.txt b/_sources/extension-guide/why-ffi.md.txt
new file mode 100644
index 00000000..e1321107
--- /dev/null
+++ b/_sources/extension-guide/why-ffi.md.txt
@@ -0,0 +1,148 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(extension_why_ffi)=
+
+# Why FFI
+
+The DataFusion in Python project is designed to allow users to extend its
+functionality in a few core areas. Ideally many users would like to package
+their extensions as a Python package and easily integrate that package with
+this project. This page describes the problem that makes the obvious approach
+fail, and the approach this project uses instead.
+
+## The primary issue
+
+Suppose you wish to use DataFusion and you have a custom data source that can
+produce tables that can then be queried against, similar to how you can
+register a {ref}`CSV <io_csv>` or {ref}`Parquet <io_parquet>` file. In
+DataFusion terminology, you likely want to implement a
+{ref}`Custom Table Provider <io_custom_table_provider>`. In an effort to make
+your data source as performant as possible and to utilize the features of
+DataFusion, you may decide to write your source in Rust and then expose it
+through [PyO3](https://pyo3.rs) as a Python library.
+
+At first glance, it may appear the best way to do this is to add the
+`datafusion-python` crate as a dependency, provide a `PyTable`, and then to
+register it with the `SessionContext`. Unfortunately, this will not work.
+
+When you produce your code as a Python library and it needs to interact with
+the DataFusion library, at the lowest level they communicate through an
+Application Binary Interface (ABI). The acronym sounds similar to API
+(Application Programming Interface), but it is distinctly different.
+
+The ABI sets the standard for how these libraries can share data and functions
+between each other. One of the key differences between Rust and other
+programming languages is that Rust does not have a stable ABI. What this means
+in practice is that if you compile a Rust library with one version of the
+`rustc` compiler and I compile another library to interface with it but I use a
+different version of the compiler, there is no guarantee the interface will be
+the same.
+
+In practice, this means that a Python library built with `datafusion-python` as
+a Rust dependency will generally **not** be compatible with the DataFusion
+Python package, even if they reference the same version of
+`datafusion-python`. If you attempt to do this, it may work on your local
+computer if you have built both packages with the same optimizations. This can
+sometimes lead to a false expectation that the code will work, but it
+frequently breaks the moment you try to use your package against the released
+packages.
+
+You can find more information about the Rust ABI in their
+[online documentation](https://doc.rust-lang.org/reference/abi.html).
+
+## The FFI approach
+
+Rust supports interacting with other programming languages through its Foreign
+Function Interface (FFI). The advantage of using the FFI is that it enables you
+to write data structures and functions that have a stable ABI. That allows you
+to use Rust code with C, Python, and other languages. In fact, the
+[PyO3](https://pyo3.rs) library uses the FFI to share data and functions
+between Python and Rust.
+
+The approach we are taking in the DataFusion in Python project is to
+incrementally expose more portions of the DataFusion project via FFI
+interfaces. This allows users to write Rust code that does **not** require the
+`datafusion-python` crate as a dependency, expose their code in Python via
+PyO3, and have it interact with the DataFusion Python package.
+
+Early adopters of this approach include
+[delta-rs](https://delta-io.github.io/delta-rs/) who has adapted their Table
+Provider for use in `datafusion-python` with only a few lines of code. Also,
+the DataFusion Python project uses the existing definitions from
+[Apache Arrow CStream Interface](https://arrow.apache.org/docs/format/CStreamInterface.html)
+to support importing **and** exporting tables. Any Python package that supports
+reading the Arrow C Stream interface can work with DataFusion Python out of the
+box! You can read more about working with Arrow sources in the
+{ref}`Data Sources <user_guide_data_sources>` page.
+
+To learn more about the Foreign Function Interface in Rust, the
+[Rustonomicon](https://doc.rust-lang.org/nomicon/ffi.html) is a good resource.
+
+## Inspiration from Arrow
+
+DataFusion is built upon [Apache Arrow](https://arrow.apache.org/). The
+canonical Python Arrow implementation,
+[pyarrow](https://arrow.apache.org/docs/python/index.html), provides an
+excellent way to share Arrow data between Python projects without performing
+any copy operations on the data, using a well defined set of interfaces — see
+their [stream interface](https://arrow.apache.org/docs/format/CStreamInterface.html).
+The [Rust Arrow implementation](https://github.com/apache/arrow-rs) also
+supports these `C` style definitions via the Foreign Function Interface. Beyond
+transferring data, `pyarrow` goes one step further and makes the interfaces
+themselves easy to share in Python, by exposing PyCapsules that contain the
+expected functionality.
+
+Two lessons we leverage from the Arrow project in DataFusion Python are:
+
+- We reuse the existing Arrow FFI functionality wherever possible.
+- We expose PyCapsules that contain an FFI stable struct.
+
+You can learn more about PyCapsules from the official
+[Python online documentation](https://docs.python.org/3/c-api/capsule.html).
+PyCapsules have excellent support in PyO3 already; the
+[PyO3 online documentation](https://pyo3.rs/main/doc/pyo3/types/struct.pycapsule)
+is a good source for more details on using PyCapsules in Rust.
+
+## If FFI does not yet cover what you need
+
+:::{note}
+Suppose you needed to expose some other features of DataFusion and you could
+not wait for the upstream repository to implement the FFI approach we describe.
+In this case you decide to create your dependency on the `datafusion-python`
+crate instead.
+
+As we discussed, this is not guaranteed to work across different compiler
+versions and optimization levels. If you wish to go down this route, there are
+two approaches we have identified you can use.
+
+1. Re-export all of `datafusion-python` yourself with your extensions built in.
+2. Carefully synchronize your software releases with the `datafusion-python` CI
+ build system so that your libraries use the exact same compiler, features,
+ and optimization level.
+
+We currently do not recommend either of these approaches as they are difficult
+to maintain over a long period. Additionally, they require a tight version
+coupling between libraries.
+
+The better path is to open an issue describing what you need exposed. The FFI
+surface in the [datafusion-ffi] crate grows in response to these.
+:::
+
+[datafusion-ffi]: https://crates.io/crates/datafusion-ffi
diff --git a/_sources/index.md.txt b/_sources/index.md.txt
index 13f7c7df..c702fc04 100644
--- a/_sources/index.md.txt
+++ b/_sources/index.md.txt
@@ -59,11 +59,22 @@ df.show()
```
+## Where to go next
+
+- **[User Guide](user-guide/index)** — reading data, building queries, tuning
+ execution, and distributing work.
+- **[Extension Guide](extension-guide/index)** — writing a library that plugs
+ into DataFusion in Python: table providers, functions, extension codecs, and
+ query planners.
+- **[Contributor Guide](contributor-guide/index)** — changing
+ datafusion-python itself.
+
```{toctree}
:hidden: true
:maxdepth: 1
user-guide/index
+extension-guide/index
contributor-guide/index
API Reference <autoapi/index>
links
diff --git a/_sources/user-guide/common-operations/aggregations.md.txt b/_sources/user-guide/common-operations/aggregations.md.txt
index 9c2d58e3..0e073185 100644
--- a/_sources/user-guide/common-operations/aggregations.md.txt
+++ b/_sources/user-guide/common-operations/aggregations.md.txt
@@ -393,11 +393,12 @@ The available aggregate functions are:
- {py:func}`datafusion.functions.regr_avgy`
- {py:func}`datafusion.functions.regr_sxx`
- {py:func}`datafusion.functions.regr_syy`
- - {py:func}`datafusion.functions.regr_slope`
+ - {py:func}`datafusion.functions.regr_sxy`
07. Positional Functions
: - {py:func}`datafusion.functions.first_value`
- {py:func}`datafusion.functions.last_value`
- {py:func}`datafusion.functions.nth_value`
+ - {py:func}`datafusion.functions.any_value`
08. String Functions
: - {py:func}`datafusion.functions.string_agg`
09. Percentile Functions
diff --git a/_sources/user-guide/common-operations/expressions.md.txt b/_sources/user-guide/common-operations/expressions.md.txt
index 7985705e..ab42172a 100644
--- a/_sources/user-guide/common-operations/expressions.md.txt
+++ b/_sources/user-guide/common-operations/expressions.md.txt
@@ -157,7 +157,9 @@ In this example, the `repeated_array` column will contain `[[1, 2, 3], [1, 2, 3]
Some array functions take a *lambda function*: a small function that runs once
per element. {py:func}`~datafusion.functions.array_transform` maps a lambda over
every element, {py:func}`~datafusion.functions.array_filter` keeps the elements
-for which a predicate lambda is true, and
+for which a predicate lambda is true,
+{py:func}`~datafusion.functions.array_first` returns the first element that
+satisfies a predicate lambda, and
{py:func}`~datafusion.functions.array_any_match` returns whether any element
satisfies a predicate lambda. (Functions that take another function as an
argument are sometimes called *higher-order* functions.)
@@ -173,6 +175,7 @@ ctx = SessionContext()
df = ctx.from_pydict({"a": [[1, 2, 3], [4, 5]]})
df.select(f.array_transform(col("a"), lambda v: v * 2).alias("doubled"))
df.select(f.array_filter(col("a"), lambda v: v > 2).alias("big_only"))
+df.select(f.array_first(col("a"), lambda v: v > 2).alias("first_big"))
df.select(f.array_any_match(col("a"), lambda v: v > 3).alias("has_big"))
```
diff --git a/_sources/user-guide/common-operations/functions.md.txt b/_sources/user-guide/common-operations/functions.md.txt
index 9939e2f0..d5ddffb7 100644
--- a/_sources/user-guide/common-operations/functions.md.txt
+++ b/_sources/user-guide/common-operations/functions.md.txt
@@ -158,3 +158,38 @@ df = df.fill_null("missing", subset=["name", "category"])
```
The fill value will be cast to match each column's type. If casting fails for a column, that column remains unchanged.
+
+## fill_nan
+
+The `fill_nan()` method replaces NaN values in floating-point columns. NaN is
+distinct from NULL, which `fill_null()` handles:
+
+```python
+# Replace NaN with 0.0 in every floating-point column
+df = df.fill_nan(0.0)
+
+# Replace NaN only in specific columns
+df = df.fill_nan(0.0, subset=["price"])
+```
+
+(fill_column_names)=
+
+## Column names with uppercase letters or dots
+
+`fill_null()` and `fill_nan()` fail on a DataFrame that has any column whose
+name contains an uppercase letter or a `.`, even when that column is not in
+`subset`
+([apache/datafusion#25829](https://github.com/apache/datafusion/issues/25829)):
+
+```python
+df = ctx.from_pydict({"Price": [float("nan"), 2.0], "qty": [float("nan"), 1.0]})
+df.fill_nan(0.0, subset=["qty"])
+# Schema error: No field named price. Did you mean '..."Price"'?
+```
+
+Until that is fixed, rename such columns first. Quote the old name so it is
+not normalized:
+
+```python
+df.with_column_renamed('"Price"', "price").fill_nan(0.0, subset=["qty"])
+```
diff --git a/_sources/user-guide/common-operations/udf-and-udfa.md.txt b/_sources/user-guide/common-operations/udf-and-udfa.md.txt
index 8d1cc876..07005fd0 100644
--- a/_sources/user-guide/common-operations/udf-and-udfa.md.txt
+++ b/_sources/user-guide/common-operations/udf-and-udfa.md.txt
@@ -391,7 +391,8 @@ df.select("a", exp_smooth(col("a")).alias("smooth_a")).show()
User Defined Table Functions are slightly different than the other functions
described here. These functions take any number of `Expr` arguments, but only
literal expressions are supported. Table functions must return a Table
-Provider as described in the ref:`_io_custom_table_provider` page.
+Provider as described in the {ref}`Custom Table Provider <io_custom_table_provider>`
+page.
Once you have a table function, you can register it with the session context
by using {py:func}`datafusion.context.SessionContext.register_udtf`.
diff --git a/_sources/user-guide/common-operations/windows.md.txt b/_sources/user-guide/common-operations/windows.md.txt
index bee96c82..31fcdbe0 100644
--- a/_sources/user-guide/common-operations/windows.md.txt
+++ b/_sources/user-guide/common-operations/windows.md.txt
@@ -135,11 +135,54 @@ df.select(
)
```
+(window_function_chaining)=
+
+#### Chaining onto a window function
+
+Set a window function's `partition_by`, `order_by`, and `window_frame` in one
+place: its keyword arguments, a single `over()`, or one builder chain ending in
+`build()`. Chaining a builder method or `over()` onto a window function that
+already has any of them raises:
+
+```python
+# Raises: lead already has window options (order_by)
+f.lead(col("v"), order_by="t").over(Window(partition_by=[col("g")]))
+
+# Set them together instead.
+f.lead(col("v")).over(Window(partition_by=[col("g")], order_by="t"))
+```
+
+A built window function stores a concrete frame with no record of whether you
+chose it or it was derived from `order_by`, so the options cannot be merged
+without guessing. Merging may become possible once
+[apache/datafusion#25934](https://github.com/apache/datafusion/issues/25934)
+is resolved.
+
+The `null_treatment` already set is kept, as are `filter` and `distinct` on an
+aggregate used as a window function. The whole-partition frame counts as no
+frame, so adding an `order_by` derives the running frame, even when you passed
+that frame explicitly:
+
+```python
+whole = WindowFrame("rows", None, None) # same as the no-order_by default
+
+# Both give a running sum.
+f.sum(col("v")).over(Window()).order_by(col("v")).build()
+f.sum(col("v")).over(Window(window_frame=whole)).order_by(col("v")).build()
+```
+
+To keep that frame, set it after the `order_by`, or pass both in one `Window`:
+
+```python
+f.sum(col("v")).over(Window()).order_by(col("v")).window_frame(whole).build()
+f.sum(col("v")).over(Window(order_by=col("v"), window_frame=whole))
+```
+
### Null Treatment
When using aggregate functions as window functions, it is often useful to specify how null values
-should be treated. In order to do this you need to use the builder function. In future releases
-we expect this to be simplified in the interface.
+should be treated. Pass `null_treatment` in the `Window`, or set it on the aggregate itself, which
+`over()` keeps (see {ref}`aggregate_over_options`).
One common usage for handling nulls is the case where you want to find the last value up to the
current row. In the following example we demonstrate how setting the null treatment to ignore
@@ -196,6 +239,30 @@ df.select(
)
```
+(aggregate_over_options)=
+
+### Options set on the aggregate
+
+`over()` keeps the `filter`, `distinct`, and `null_treatment` options an
+aggregate was built with:
+
+```python
+# Averages the distinct values 1.0 and 4.0.
+f.avg(col("v"), distinct=True).over(Window())
+```
+
+An aggregate's `order_by` raises, as `ORDER BY` inside an aggregate call does
+with `OVER` in SQL. A window does not pass an ordering to the aggregate, so
+the `order_by` in the `Window` only sets the frame and the order of rows. The
+one exception is a `WITHIN GROUP` function such as
+{py:func}`~datafusion.functions.percentile_cont`, which computes ascending as a
+window: an ascending `sort_expression` is accepted, and a descending one raises.
+
+```python
+f.percentile_cont(col("v"), 0.25).over(Window()) # ascending, accepted
+f.percentile_cont(col("v").sort(ascending=False), 0.25).over(Window()) # raises
+```
+
## Available Functions
The possible window functions are:
diff --git a/_sources/user-guide/configuration.md.txt b/_sources/user-guide/configuration.md.txt
index d1c5c9b4..a5e926d0 100644
--- a/_sources/user-guide/configuration.md.txt
+++ b/_sources/user-guide/configuration.md.txt
@@ -49,6 +49,38 @@ ctx = SessionContext(config, runtime)
print(ctx)
```
+## Setting options by key
+
+The `with_*` methods cover the common options, but any option DataFusion declares can be
+set by its fully qualified key with {py:meth}`~datafusion.SessionConfig.set`. The value is
+always a string, and is parsed according to the type the option declares, so an unknown key
+or an unparsable value raises rather than being silently ignored:
+
+```python
+config = SessionConfig().set("datafusion.execution.batch_size", "1024")
+```
+
+A whole dictionary of options can be applied at once by passing it to the
+{py:class}`~datafusion.SessionConfig` constructor, which is the shape a replayed set of
+settings usually arrives in:
+
+```python
+config = SessionConfig({"datafusion.execution.batch_size": "1024"})
+```
+
+Both routes reject the same keys, so which one you use does not change what is accepted. The
+constructor applies its entries in an unspecified order, so a dictionary with more than one
+bad key does not report a predictable one first.
+
+One trap is worth knowing about if you read settings back out of a session and replay them
+somewhere else, such as onto a worker process or into a test fixture. With
+`with_information_schema(True)`, the `information_schema.df_settings` table lists the
+`datafusion.runtime.*` keys alongside the rest, but those come from the runtime environment
+rather than from `ConfigOptions` and cannot be set this way. Feeding that table's rows back
+in verbatim will fail on the first such row, whichever route you use. Configure the runtime
+through `RuntimeEnvBuilder` instead, and skip the `datafusion.runtime.` prefix when
+replaying.
+
## Maximizing CPU Usage
DataFusion uses partitions to parallelize work. For small queries the
@@ -96,6 +128,58 @@ df = df.repartition_by_hash(col("a"), num=16)
result = df.collect()
```
+(checking_partitioning)=
+
+### Checking what the plan actually does
+
+`repartition` and `repartition_by_hash` are requests, not instructions. The optimizer is
+free to drop a repartition nothing downstream needs, to collapse partitions again for an
+operator that requires a single stream, or to substitute a repartition of its own sized by
+`target_partitions`. So the number you passed is not necessarily the number you get.
+
+{py:attr}`~datafusion.ExecutionPlan.output_partitioning` reports what the built plan does,
+as opposed to what was asked of it:
+
+```python
+from datafusion import SessionConfig, SessionContext, col, functions as f
+
+config = SessionConfig().with_target_partitions(16)
+ctx = SessionContext(config)
+
+df = ctx.read_parquet("data.parquet").repartition_by_hash(col("a"), num=8)
+plan = df.aggregate([col("a")], [f.sum(col("b"))]).execution_plan()
+
+partitioning = plan.output_partitioning
+print(partitioning.scheme) # 'Hash'
+print(partitioning.partition_count) # 16 -- target_partitions, not the 8 requested
+print(partitioning.hash_expressions) # ['a@0']
+```
+
+The request for eight partitions did not survive: the optimizer inserted its own hash
+repartition at `target_partitions` instead. Had the aggregation been left off, the
+repartition would have been removed altogether and the plan would report
+`UnknownPartitioning` over the source's own partition count.
+
+`UnknownPartitioning` means the plan knows how many partitions it has but nothing about how
+rows are distributed across them, which is the ordinary case for a file scan.
+{py:attr}`~datafusion.ExecutionPlan.partition_count` gives the same count on its own when
+the scheme does not matter.
+
+Four schemes exist, but only three of them can come out of a plan you built here.
+`UnknownPartitioning` comes from a source, and `RoundRobinBatch` and `Hash` from a
+repartition — either one you asked for or one the optimizer inserted. `Range`, which spreads
+an ordered key space across partitions at chosen split points, has no request form in this
+package: `repartition` asks for round-robin, `repartition_by_hash` asks for hash, and SQL
+has no range-repartition syntax.
+
+It is still worth handling, because a plan does not have to have been built here. Both
+`datafusion-proto` and `datafusion-ffi` carry range partitioning faithfully, so
+{py:meth}`~datafusion.ExecutionPlan.from_bytes` can return a plan reporting it, as can an
+extension library whose query planner builds one — see {ref}`extension_planners`. Such a
+plan executes normally; only the split points are invisible, since
+{py:attr}`~datafusion.PhysicalPartitioning.hash_expressions` returns `None` for every scheme
+but `Hash`. Read {py:func}`repr` of the partitioning to see them.
+
### Benchmark Example
The repository includes a benchmark script that demonstrates how to maximize CPU usage
diff --git a/_sources/user-guide/data-sources.md.txt b/_sources/user-guide/data-sources.md.txt
index 22e66683..ad5718ed 100644
--- a/_sources/user-guide/data-sources.md.txt
+++ b/_sources/user-guide/data-sources.md.txt
@@ -89,7 +89,9 @@ ctx.create_dataframe([[batch]]).show()
## Object Store
DataFusion has support for multiple storage options in addition to local files.
-The example below requires an appropriate S3 account with access credentials.
+The example below requires access to the S3 bucket. Set `AWS_ACCESS_KEY_ID` and
+`AWS_SECRET_ACCESS_KEY` in the environment before running it. For temporary
+credentials, also set `AWS_SESSION_TOKEN`.
Supported Object Stores are
@@ -100,16 +102,17 @@ Supported Object Stores are
- {py:class}`~datafusion.object_store.MicrosoftAzure`
```python
+from datafusion import SessionContext
from datafusion.object_store import AmazonS3
region = "us-east-1"
bucket_name = "yellow-trips"
+ctx = SessionContext()
+
s3 = AmazonS3(
bucket_name=bucket_name,
region=region,
- access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
- secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
)
path = f"s3://{bucket_name}/"
@@ -120,6 +123,27 @@ ctx.register_parquet("trips", path)
ctx.table("trips").show()
```
+### Query S3 data with SQL
+
+To reach the same data from SQL, give the S3 path a table name with
+`CREATE EXTERNAL TABLE`. The registered object store carries the credentials and
+the region, so the statement itself needs only the location.
+
+Register the store for the bucket as shown above, then use that same
+{py:class}`~datafusion.context.SessionContext` to create and query the table:
+
+```python
+ctx.sql(
+ f"""
+ CREATE EXTERNAL TABLE trips_sql
+ STORED AS PARQUET
+ LOCATION '{path}'
+ """
+).collect()
+
+ctx.sql("SELECT count(passenger_count) FROM trips_sql").show()
+```
+
## Other DataFrame Libraries
DataFusion can import DataFrames directly from other libraries, such as
@@ -214,7 +238,7 @@ Features that are available in PyIceberg but not yet in Iceberg Rust will not be
## Custom Table Provider
You can implement a custom Data Provider in Rust and expose it to DataFusion through the
-the interface as describe in the {ref}`Custom Table Provider <io_custom_table_provider>`
+interface described in the {ref}`Custom Table Provider <io_custom_table_provider>`
section. This is an advanced topic, but a
[user example](https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-example)
is provided in the DataFusion repository.
diff --git a/_sources/user-guide/distributing-work.md.txt b/_sources/user-guide/distributing-work/expressions.md.txt
similarity index 86%
rename from _sources/user-guide/distributing-work.md.txt
rename to _sources/user-guide/distributing-work/expressions.md.txt
index 527fe883..252defc2 100644
--- a/_sources/user-guide/distributing-work.md.txt
+++ b/_sources/user-guide/distributing-work/expressions.md.txt
@@ -17,24 +17,19 @@
under the License.
-->
-# Distributing work
+(distributed_expressions)=
+
+# Shipping expressions to workers
DataFusion supports splitting work across processes by shipping
serialized expressions to workers: the driver builds an
{py:class}`~datafusion.Expr`, each worker evaluates it against its
own slice of data. This pattern suits embarrassingly-parallel
-workloads where the driver decides partitioning up front.
-
-Query-level distribution — where the runtime partitions a single
-logical or physical plan across worker nodes — is in progress
-upstream via [datafusion-distributed](https://github.com/apache/datafusion-distributed) and [Apache
-Ballista](https://github.com/apache/datafusion-ballista). Both
-have short sections at the end of this page; integration details
-will land as those projects become usable from datafusion-python.
-
-## Expression-level distribution
+workloads where the driver decides partitioning up front. If you
+would rather have a library partition your query plan for you, see
+{ref}`distributed_query_engines` instead.
-DataFusion expressions support distribution directly: pass one to a
+Expressions support this directly: pass one to a
worker process and Python's standard
[pickle](https://docs.python.org/3/library/pickle.html) machinery
serializes it transparently — the same machinery
@@ -43,7 +38,7 @@ similar libraries already use to ship function arguments. Python UDFs
— scalar, aggregate, and window — travel inside the serialized
expression; the receiver does not need to pre-register them.
-### Basic worker-pool example
+## Basic worker-pool example
Define a worker function that takes the expression plus a batch and
returns the evaluated result:
@@ -92,7 +87,7 @@ see [Safe importing of main module](https://docs.python.org/3/library/multiproce
in the Python docs.
:::
-### What travels with the expression
+## What travels with the expression
- **Built-in functions** (`abs`, `length`, arithmetic, comparisons,
etc.) — fully portable. Worker needs nothing pre-registered.
@@ -113,7 +108,7 @@ in the Python docs.
(distributed_udf_portability)=
-### Portability requirements for inline Python UDFs
+## Portability requirements for inline Python UDFs
Inline Python UDFs ride on [cloudpickle](https://github.com/cloudpipe/cloudpickle), which imposes two
requirements on the worker environment:
@@ -133,7 +128,7 @@ requirements on the worker environment:
Self-contained UDFs (no imports beyond what the worker already has,
e.g. `pyarrow`) avoid this entirely.
-### Registering shared UDFs on workers
+## Registering shared UDFs on workers
When an expression references an FFI capsule UDF (or any UDF the
worker must resolve from its registered functions), set up the
@@ -164,7 +159,7 @@ fine for expressions that only reference built-ins and Python UDFs,
but FFI-capsule-backed registrations must be installed on the global
context to resolve.
-### Python 3.14 default change
+## Python 3.14 default change
Python 3.14 changed the Linux default start method for
{py:mod}`multiprocessing` from `fork` to `forkserver` (macOS has
@@ -174,7 +169,7 @@ workers via copy-on-write; with `forkserver` and `spawn` it is
not. The {py:func}`~datafusion.ipc.set_worker_ctx` pattern works on
every start method — prefer it over relying on inherited state.
-### Practical considerations
+## Practical considerations
- **Serialized size scales with what travels inline.** A serialized
expression of just built-ins is small (tens of bytes). An
@@ -189,7 +184,7 @@ every start method — prefer it over relying on inherited state.
the captured state is large, mutable, or not portable to the
worker's environment. See {ref}`Portability requirements for inline Python UDFs <distributed_udf_portability>` for the Python-version and imported-module rules.
-### Disabling Python UDF inlining
+## Disabling Python UDF inlining
For a stricter wire format, call
{py:meth}`SessionContext.with_python_udf_inlining(enabled=False)
@@ -241,7 +236,7 @@ threat model.
(distributed_expr_security)=
-### Security
+## Security
:::{warning}
Reconstructing an expression containing a Python UDF executes
@@ -255,7 +250,9 @@ functions and pre-registered Rust-side UDFs, and avoid
{py:func}`pickle.loads` on externally supplied bytes entirely.
:::
-### Reference: session context slots
+(session_context_slots)=
+
+## Reference: session context slots
There is only one type — {py:class}`SessionContext`. It can occupy
up to four *slots* in a running program:
@@ -297,32 +294,10 @@ Sharp edges:
- The inlining toggle is per-context state, not a global switch.
Two contexts with different toggles can coexist in one process.
-## Query-level distribution via datafusion-distributed
-
-🚧 *Work in progress upstream — not yet usable from datafusion-python.*
-
-[datafusion-distributed](https://github.com/apache/datafusion-distributed)
-splits a single physical plan into stages and runs each stage on a
-different worker node. The driver writes a SQL or DataFrame query
-once; the runtime handles partitioning, shuffles, and reassembly.
-
-A datafusion-python integration is in development. This section will
-document the integration once it lands. In the meantime, the
-expression-level approach above covers most use cases that do not
-require automatic plan partitioning.
-
-## Query-level distribution via Apache Ballista
-
-🚧 *Work in progress upstream — not yet usable from datafusion-python.*
-
-[Apache Ballista](https://github.com/apache/datafusion-ballista)
-provides distributed query execution on top of DataFusion with a
-scheduler / executor model better suited to long-lived cluster
-deployments. A datafusion-python integration is on the roadmap; this
-section will fill in once the integration is usable.
-
## See also
+- {ref}`distributed_query_engines` — the other road: let a library
+ partition the plan for you.
- {py:mod}`datafusion.ipc` — worker context API.
- `examples/multiprocessing_pickle_expr.py` — runnable
`multiprocessing.Pool` example that ships a different parametric
diff --git a/_sources/user-guide/distributing-work/index.md.txt b/_sources/user-guide/distributing-work/index.md.txt
new file mode 100644
index 00000000..3047ea39
--- /dev/null
+++ b/_sources/user-guide/distributing-work/index.md.txt
@@ -0,0 +1,81 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(distributing_work)=
+
+# Distributing work
+
+A single {py:class}`~datafusion.SessionContext` already uses every
+core on the machine — DataFusion partitions and parallelizes within
+a process without being asked. See
+{doc}`../configuration` for tuning that.
+
+This section is about the step after that: getting work onto more
+than one process or more than one machine. There are two roads, and
+they suit different problems.
+
+## Pick your road
+
+**You decide the partitioning → ship expressions.**
+
+You already know how the data divides — one file per worker, one
+customer per worker, one parameter setting per worker. You build an
+{py:class}`~datafusion.Expr` in the driver and hand a copy to each
+worker along with its slice. Standard Python `pickle` moves it, so
+{py:mod}`multiprocessing`, Ray, and anything else that ships function
+arguments works with no extra machinery.
+
+Best for embarrassingly-parallel work: parameter sweeps, per-file
+transforms, scoring batches. Available today.
+
+→ {ref}`distributed_expressions`
+
+**A library decides the partitioning → install a query engine.**
+
+You write one ordinary SQL or DataFrame query against a table too
+large for one machine, and an engine library splits the plan into
+stages, runs them on its workers, and reassembles the result. You do
+not partition anything and your queries do not change.
+
+Best for large-scale analytical queries — joins and aggregations over
+data that does not fit on one node. Being built upstream; not yet
+usable from datafusion-python.
+
+→ {ref}`distributed_query_engines`
+
+## Choosing between them
+
+The two are not competing implementations of one feature. The
+question is who owns the partitioning decision.
+
+If you can state the partitioning in one line — "one worker per input
+file" — the expression road is simpler, has no cluster to operate,
+and works now. If stating it requires knowing how a join will
+shuffle, that decision belongs to a query planner, which is what an
+engine library provides.
+
+They also compose. An engine handles the query; expressions handle
+whatever you want to fan out around it.
+
+```{toctree}
+:maxdepth: 2
+
+expressions
+query-engines
+```
diff --git a/_sources/user-guide/distributing-work/query-engines.md.txt b/_sources/user-guide/distributing-work/query-engines.md.txt
new file mode 100644
index 00000000..0aa045c7
--- /dev/null
+++ b/_sources/user-guide/distributing-work/query-engines.md.txt
@@ -0,0 +1,100 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(distributed_query_engines)=
+
+# Distributed query engines
+
+A distributed query engine takes the query you already wrote and runs
+it across several machines for you. You do not partition anything by
+hand and you do not change your queries — the engine installs itself
+on your session, rewrites the plan into stages, and executes those
+stages on its own workers.
+
+This is the counterpart to {ref}`distributed_expressions`, where
+*you* decide the partitioning and ship one expression per slice of
+data. Choose an engine when you want a single query spread across a
+cluster; choose expressions when you already know how to split the
+work and only need parallelism.
+
+## How an engine attaches to your session
+
+An engine library ships an object you hand to
+{py:meth}`~datafusion.SessionContext.with_extensions`:
+
+```python
+from datafusion import SessionContext
+import my_engine
+
+ctx = SessionContext().with_extensions(my_engine.Extension("scheduler:50050"))
+
+ctx.register_table("events", my_engine.TableProvider("s3://bucket/events"))
+ctx.sql("SELECT country, count(*) FROM events GROUP BY country").show()
+```
+
+The `sql` call is unchanged from a single-process program. What
+changed is underneath it: installing the engine gave the session a
+query planner of the engine's own, and that planner is what turns
+your plan into distributed stages.
+
+Two consequences worth knowing:
+
+- **The engine must be installed before you run the query**, not
+ before you register tables. Registration order does not matter;
+ `with_extensions` is a session-level setup step.
+- **The engine has to be able to reach your tables and functions.**
+ It ships the plan to its workers, so anything the plan references
+ has to be reconstructible there. Table providers from the engine's
+ own library always are. A Python UDF may or may not be — the engine
+ library documents what it supports, and
+ {ref}`distributed_udf_portability` describes the constraints that
+ apply to Python callables crossing a process boundary in general.
+
+If you install more than one library, pass them in one
+`with_extensions` call so they can see each other. See
+{ref}`user_guide_extensions` for the details of installing extension
+libraries, and {ref}`ffi` if you want to write an engine yourself.
+
+## Available engines
+
+Query-level distribution is being built upstream. Neither project
+below is usable from datafusion-python yet; both sections will fill
+in as the integrations land.
+
+### datafusion-distributed
+
+🚧 *Work in progress upstream — not yet usable from datafusion-python.*
+
+[datafusion-distributed](https://github.com/apache/datafusion-distributed)
+splits a single physical plan into stages and runs each stage on a
+different worker node. The driver writes a SQL or DataFrame query
+once; the runtime handles partitioning, shuffles, and reassembly.
+
+A datafusion-python integration is in development. In the meantime,
+{ref}`distributed_expressions` covers most use cases that do not
+require automatic plan partitioning.
+
+### Apache Ballista
+
+🚧 *Work in progress upstream — not yet usable from datafusion-python.*
+
+[Apache Ballista](https://github.com/apache/datafusion-ballista)
+provides distributed query execution on top of DataFusion with a
+scheduler / executor model better suited to long-lived cluster
+deployments. A datafusion-python integration is on the roadmap.
diff --git a/_sources/user-guide/extensions.md.txt b/_sources/user-guide/extensions.md.txt
new file mode 100644
index 00000000..8c86b4a6
--- /dev/null
+++ b/_sources/user-guide/extensions.md.txt
@@ -0,0 +1,142 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+(user_guide_extensions)=
+
+# Using extension libraries
+
+An extension library is a separate package that teaches a
+{py:class}`~datafusion.SessionContext` something it does not know on its own —
+a new data source, extra functions, or a different way of executing your
+queries. You install it with `pip`, hand it to your session, and keep writing
+the same SQL and DataFrame code.
+
+Examples in the wild include [delta-rs](https://delta-io.github.io/delta-rs/),
+which exposes Delta Lake tables to DataFusion, and the two worked examples in
+this repository under
+[`examples/`](https://github.com/apache/datafusion-python/tree/main/examples).
+
+## Two kinds of extension
+
+Which one you have determines how much setup you do.
+
+**Tables and functions register directly.** If the library gives you a table
+or a function, register it the same way you would register a CSV file. No
+extra setup:
+
+```python
+from datafusion import SessionContext
+import my_tables
+
+ctx = SessionContext()
+ctx.register_table("events", my_tables.TableProvider("s3://bucket/events"))
+ctx.sql("SELECT count(*) FROM events").show()
+```
+
+**Libraries that change how queries run need to be installed on the session.**
+A distributed engine, or anything that rewrites your query plan, has to be
+attached to the session before it can do its work. That is what
+{py:meth}`~datafusion.SessionContext.with_extensions` is for. The library
+documents an object — often called `Extension` — that you pass to it:
+
+```python
+from datafusion import SessionContext
+import my_engine
+
+ctx = SessionContext().with_extensions(my_engine.Extension("scheduler:50050"))
+ctx.register_table("events", my_engine.TableProvider("s3://bucket/events"))
+ctx.sql("SELECT count(*) FROM events").show()
+```
+
+`with_extensions` returns a context; use the returned one. It shares
+everything else with the context you called it on, so tables you registered
+before the call are still there.
+
+## Using more than one library
+
+Pass them all to a single call:
+
+```python
+ctx = SessionContext().with_extensions(
+ my_tables.Extension(),
+ my_engine.Extension("scheduler:50050"),
+)
+```
+
+One call rather than several is worth preferring: it lets the libraries see
+each other, which they cannot do if you install them one at a time. Order
+rarely matters. When a library needs a particular position — usually "list me
+last" for something that wraps the others — it says so in its own
+documentation.
+
+## Two things that will bite you
+
+**Keep your context alive.** A `DataFrame` or a plan does not keep its session
+alive on its own. If a context is garbage-collected while something built from
+it is still in use, the next query fails with:
+
+```text
+TaskContextProvider went out of scope over FFI boundary
+```
+
+Almost always this is a helper that built a context locally and returned a
+DataFrame:
+
+```python
+# Wrong — ctx is collected when the function returns.
+def load():
+ ctx = SessionContext().with_extensions(my_engine.Extension())
+ return ctx.sql("SELECT * FROM events")
+
+# Right — hand back the context too, or keep it on an object that lives
+# as long as the frames derived from it.
+def load():
+ ctx = SessionContext().with_extensions(my_engine.Extension())
+ return ctx, ctx.sql("SELECT * FROM events")
+```
+
+**Versions have to match.** An extension library is compiled against one
+DataFusion version. A mismatch raises an `ImportError` naming the version it
+found and the version expected, at the moment you register or install the
+library — not silently at query time. If you see one, upgrade or downgrade the
+extension library so its DataFusion version matches this package's. See
+{ref}`extension_version_mismatch`.
+
+## Checking what a session knows about
+
+{py:meth}`~datafusion.SessionContext.logical_extension_codec_ids` and
+{py:meth}`~datafusion.SessionContext.physical_extension_codec_ids` list which
+libraries a session has been taught about. Useful when a query fails and you
+want to confirm the library actually got installed:
+
+```python
+ctx = SessionContext().with_extensions(my_engine.Extension())
+ctx.logical_extension_codec_ids()
+# ['my_engine.LogicalCodec']
+```
+
+An empty list means nothing extra is installed.
+
+## Next steps
+
+- {ref}`distributed_query_engines` — running your queries across several
+ machines with an engine library.
+- {ref}`distributed_expressions` — the other road to parallelism, where you
+ decide the partitioning and ship expressions to a worker pool.
+- {ref}`ffi` — writing an extension library of your own.
diff --git a/_sources/user-guide/index.md.txt b/_sources/user-guide/index.md.txt
index 0155c4e9..07b912a8 100644
--- a/_sources/user-guide/index.md.txt
+++ b/_sources/user-guide/index.md.txt
@@ -20,7 +20,11 @@
# User Guide
The user guide walks through installing DataFusion in Python, building queries
-with the DataFrame API or SQL, reading and writing data, and tuning execution.
+with the DataFrame API or SQL, reading and writing data, tuning execution, and
+spreading work across processes or machines.
+
+If you are writing an extension library rather than using one, see the
+{ref}`Extension Guide <ffi>`.
```{toctree}
:maxdepth: 2
@@ -32,7 +36,8 @@ dataframe/index
common-operations/index
io/index
configuration
-distributing-work
+extensions
+distributing-work/index
sql
upgrade-guides
ai-coding-assistants
diff --git a/_sources/user-guide/io/table_provider.md.txt b/_sources/user-guide/io/table_provider.md.txt
index 3c436ba1..1b6987e7 100644
--- a/_sources/user-guide/io/table_provider.md.txt
+++ b/_sources/user-guide/io/table_provider.md.txt
@@ -28,24 +28,8 @@ you must use DataFusion 43.0.0 or later and expose a [FFI_TableProvider](https:/
via [PyCapsule](https://pyo3.rs/main/doc/pyo3/types/struct.pycapsule).
A complete example can be found in the [examples folder](https://github.com/apache/datafusion-python/tree/main/examples).
-
-```rust
-#[pymethods]
-impl MyTableProvider {
-
- fn __datafusion_table_provider__<'py>(
- &self,
- py: Python<'py>,
- ) -> PyResult<Bound<'py, PyCapsule>> {
- let name = cr"datafusion_table_provider".into();
-
- let provider = Arc::new(self.clone());
- let provider = FFI_TableProvider::new(provider, false, None);
-
- PyCapsule::new_bound(py, provider, Some(name.clone()))
- }
-}
-```
+For how to write one — the getter, what it receives, and how to serialize what
+it exposes — see {ref}`extension_providers` in the Extension Guide.
Once you have this library available, you can construct a
{py:class}`~datafusion.Table` in Python and register it with the
diff --git a/_sources/user-guide/upgrade-guides.md.txt b/_sources/user-guide/upgrade-guides.md.txt
index 360e0533..74318c83 100644
--- a/_sources/user-guide/upgrade-guides.md.txt
+++ b/_sources/user-guide/upgrade-guides.md.txt
@@ -19,6 +19,423 @@
# Upgrade Guides
+## DataFusion 56.0.0
+
+### `sort` and `order_by` now order `NULLS LAST` by default
+
+Calling `.sort(...)` on a DataFrame or using the `.order_by(...)` function now
+orders rows with `NULLS LAST` by default, instead of `NULLS FIRST`. This follows
+the same behavior as SQL's `ORDER BY` or the DataFrame's `sort_by(...)`.
+
+To go back to the previous behavior, we need to explicitly specify
+`nulls_first=True`. Example:
+
+```python
+# sort
+df.sort(column("a"), nulls_first=True)
+
+# order by
+expr = f.first_value(column("a")).over(
+ Window(
+ partition_by=partition,
+ order_by=f.order_by(column("b"), nulls_first=True)
+ )
+)
+```
+
+## DataFusion 55.0.0
+
+This release extends the change made in 52.0.0 to the remaining
+{ref}`extension_capsule_protocol` hook methods. Users who contribute their own
+`LogicalExtensionCodec` or
+`PhysicalExtensionCodec` via FFI must update
+`__datafusion_logical_extension_codec__` and
+`__datafusion_physical_extension_codec__` to accept an additional
+`session: Bound<PyAny>` parameter, and take the `TaskContextProvider` from that
+session rather than constructing a `SessionContext` of their own.
+
+Before:
+
+```rust
+fn __datafusion_physical_extension_codec__<'py>(
+ &self,
+ py: Python<'py>,
+) -> PyResult<Bound<'py, PyCapsule>> {
+ let ctx_provider: Arc<dyn TaskContextProvider> = Arc::clone(&self.ctx_provider);
+ let ffi = FFI_PhysicalExtensionCodec::new(inner, Some(runtime), &ctx_provider);
+ PyCapsule::new_with_value(py, ffi, cr"datafusion_physical_extension_codec")
+}
+```
+
+After:
+
+```rust
+fn __datafusion_physical_extension_codec__<'py>(
+ &self,
+ py: Python<'py>,
+ session: Bound<'py, PyAny>,
+) -> PyResult<Bound<'py, PyCapsule>> {
+ let ctx_provider = ffi_task_context_provider_from_pycapsule(&session)?;
+ let ffi = FFI_PhysicalExtensionCodec::new(inner, Some(runtime), ctx_provider);
+ PyCapsule::new_with_value(py, ffi, cr"datafusion_physical_extension_codec")
+}
+```
+
+The dropped `&` on the last argument is not a typo. That parameter is
+`impl Into<FFI_TaskContextProvider>`, so it accepts either an
+`&Arc<dyn TaskContextProvider>`, as before, or an `FFI_TaskContextProvider`,
+which is what `ffi_task_context_provider_from_pycapsule` hands back. Both forms
+compile; the argument changes because the provider now comes from the session
+rather than from a field.
+
+A codec that keeps its own `SessionContext` still compiles, but its decode
+callbacks resolve names against that empty session instead of the one running
+the query, so a function registered with `SessionContext.register_udf` is not
+visible to it. Taking the provider from `session` also removes a lifetime
+hazard: `FFI_TaskContextProvider` holds its provider weakly, so a context
+constructed inside the getter is already dropped by the time the capsule is
+used.
+
+`SessionContext` accepts the argument on its own capsule getters and ignores
+it, so existing calls such as `ctx.__datafusion_logical_extension_codec__()`
+continue to work unchanged.
+
+The provider and catalog getters —
+`__datafusion_table_provider_factory__`, `__datafusion_catalog_provider__`,
+`__datafusion_catalog_provider_list__`, and `__datafusion_schema_provider__` —
+now receive the host's logical extension codec as a bare
+`datafusion_logical_extension_codec` capsule rather than a session.
+`__datafusion_table_provider__` receives a session from
+`SessionContext.register_table` and a codec capsule from
+`Schema.register_table`. No code change is needed in either case: pass the
+argument to `ffi_logical_codec_from_pycapsule`, which calls the codec getter
+when the object has one and returns the object unchanged when it does not. Do
+not inspect the argument's type. See {ref}`extension_getter_argument`.
+
+New in this release, `__datafusion_query_planner__` follows the same protocol.
+It receives the session and takes both extension codecs from it, so a planner
+library never builds a `TaskContextProvider` at all. Install one with
+`SessionContext.set_query_planner(planner)`, which mutates the session the same
+way `add_physical_optimizer_rule` does and returns nothing — the query planner
+lives in `SessionState`, so it belongs to the session rather than to a
+particular handle on it. See {ref}`extension_planners` for the full protocol.
+
+(extension_version_mismatch)=
+
+### Mismatched extension libraries now fail loudly
+
+Objects imported through the capsule protocol are checked against the major
+version of `datafusion-ffi` this package was built with. A table provider,
+extension codec, or query planner produced by a library built against a
+different DataFusion major version now raises an `ImportError` naming the
+version found and the one expected, instead of being used as-is. Table
+providers previously performed no such check.
+
+This is a diagnostic rather than a soundness guarantee — reading the version
+out of the struct already assumes the local field layout — but it turns the
+common "extension library built against the wrong DataFusion" mistake into a
+clear message rather than undefined behaviour on first use.
+
+`FFI_TaskContextProvider`, `FFI_TableProviderFactory`, and `FFI_ExtensionOptions`
+carry no version field, so objects of those types cannot be checked.
+
+### Extension codecs compose instead of replacing
+
+`SessionContext.with_logical_extension_codec` and
+`with_physical_extension_codec` previously replaced whichever codec was already
+installed, so a session could only ever have one. Installing a second codec
+silently discarded the first, and plans failed later with a confusing decode
+error. Both methods now append to a chain, and a session can carry codecs from
+several independent libraries at once.
+
+**No change is required in an extension codec.** Keep implementing
+`LogicalExtensionCodec` or `PhysicalExtensionCodec` exactly as before. Your codec
+is still handed back exactly the bytes it wrote, and is never handed a payload
+another library's codec wrote.
+
+Callers relying on replacement semantics — installing a codec in order to remove
+a previous one — are affected. There is no way to remove an installed codec.
+
+A serialized plan now records which codec wrote each payload, as a short id taken
+from the codec's class. Two behaviours follow from that:
+
+- Installing two instances of one class raises a `ValueError`, because both would
+ claim the same id. Pass `codec_id=` to tell them apart.
+- A codec installed from a bare `PyCapsule` has no class to take an id from, so it
+ gets one private to the session that installed it. It works normally on that
+ session, but a plan it encodes cannot be decoded on an unrelated one. Pass
+ `codec_id=` if those plans have to cross sessions.
+
+```python
+ctx = ctx.with_logical_extension_codec(lib_a.codec())
+ctx = ctx.with_logical_extension_codec(lib_b.codec()) # no longer discards lib_a
+
+# Two instances of one class need distinct ids.
+ctx = ctx.with_logical_extension_codec(lib_a.Codec(), codec_id="lib_a.reader")
+ctx = ctx.with_logical_extension_codec(lib_a.Codec(), codec_id="lib_a.writer")
+
+ctx.logical_extension_codec_ids()
+```
+
+Serialized plans change shape once an extension codec is installed, because each
+payload now records which codec wrote it. A session with no extension codecs
+installed produces the same bytes as before, as do functions encoded by name.
+Regenerate any plan you serialized with an earlier release and stored for later
+use, if it was produced by a session with an extension codec installed.
+
+### Capsule-getter protocols moved to `datafusion.extensions`
+
+`PhysicalOptimizerRuleExportable` now lives in `datafusion.extensions`, next to
+the other protocols an extension library implements against. It was previously
+importable from `datafusion.context`, and that path is gone.
+
+```python
+from datafusion.context import PhysicalOptimizerRuleExportable # before
+from datafusion.extensions import PhysicalOptimizerRuleExportable # after
+```
+
+This affects type annotations only. The protocol is structural and not
+`@runtime_checkable`, so nothing imports it to call `isinstance`, and
+`SessionContext.add_physical_optimizer_rule` is unchanged — a rule object that
+worked before still works, whether or not its library names the protocol
+anywhere.
+
+The bundle protocols added in this release — `QueryPlannerExportable`,
+`SessionComponentsExportable`, and `SessionPlannerExportable` — are reached the
+same way, through `datafusion.extensions` rather than the package root. They are
+new in 55.0.0, so no earlier import path existed. `SessionExtensionComponents`
+stays at the root, because a bundle constructs one rather than merely naming it:
+
+```python
+from datafusion import SessionExtensionComponents
+from datafusion.extensions import SessionComponentsExportable
+```
+
+### `SessionContext.execute` renamed its second parameter
+
+The parameter is a single partition index, not a count, and is now named
+`partition` rather than `partitions`. Positional calls are unaffected; update
+any call passing it by keyword.
+
+```python
+ctx.execute(plan, partitions=0) # before
+ctx.execute(plan, partition=0) # after
+```
+
+### More aggregate functions accept `distinct`
+
+{py:func}`~datafusion.functions.bit_and`,
+{py:func}`~datafusion.functions.bit_or`,
+{py:func}`~datafusion.functions.mean`,
+{py:func}`~datafusion.functions.percentile_cont`,
+{py:func}`~datafusion.functions.quantile_cont`, and
+{py:func}`~datafusion.functions.string_agg` now accept a `distinct` argument.
+As with `sum` and `avg` in 54.0.0, `distinct` is inserted *before* `filter`, so
+code that passed `filter` (or, for `string_agg`, `order_by`) positionally must
+pass it by keyword.
+
+```python
+f.bit_and(column("a"), my_filter) # before
+f.bit_and(column("a"), filter=my_filter) # after
+```
+
+Passing `filter` to `mean` previously raised a `TypeError`, whether passed
+positionally or by keyword; it now works when passed by keyword.
+
+### Chaining keeps options already set
+
+Chaining a builder method (`order_by`, `filter`, `distinct`, `null_treatment`,
+`partition_by`, `window_frame`) or `over()` onto a function used to start from
+an empty builder, so options set by the function's keyword arguments were
+silently reset. On an aggregate they are now kept, which can change results:
+
+```python
+e = f.string_agg(col("s"), ",", order_by="s")
+e.distinct().build() # before: order_by dropped; after: kept
+```
+
+On a window function that already has a `partition_by`, `order_by`, or
+`window_frame`, chaining now raises instead of dropping them. Set them in one
+place, as described in {ref}`window_function_chaining`:
+
+```python
+e = f.lead(col("v"), 1, partition_by=[col("g")], order_by="t")
+e.over(Window(order_by="t")) # before: partition dropped; after: raises
+f.lead(col("v"), 1).over(Window(partition_by=[col("g")], order_by="t")) # after
+```
+
+Options that used to be dropped now take effect, so a chain that ran before
+may now raise. For example, DISTINCT requires the ORDER BY expressions to be
+among the arguments:
+
+```python
+f.array_agg(col("s"), distinct=True).order_by(col("v")).build()
+# before: ran without DISTINCT
+# after: Execution error: In an aggregate with DISTINCT, ORDER BY expressions
+# must appear in argument list
+```
+
+Drop `distinct`, or order by the aggregated column, to get either of the
+results the chain can actually produce.
+
+An option that does not apply to the function now raises as soon as it is
+set, anywhere in the chain. `filter` and `distinct` need an aggregate,
+including one used as a window function, and `partition_by` and
+`window_frame` need a window function. Later in a chain these were silently
+dropped:
+
+```python
+f.sum(col("v")).filter(col("v") > lit(1)).partition_by(col("g"))
+# before: partition_by dropped; after: raises
+```
+
+The default `RESPECT NULLS` set by `first_value`, `last_value`, and `nth_value`
+is also kept, so their generated column names change:
+
+```python
+f.first_value(col("a")).order_by(col("b")).build()
+# before: first_value(a) ORDER BY [b ASC NULLS FIRST]
+# after: first_value(a) RESPECT NULLS ORDER BY [b ASC NULLS FIRST]
+```
+
+This now matches the name from `f.first_value(col("a"), order_by=col("b"))`.
+Code that selects the result by its generated name should `alias()` it instead.
+
+### `over()` keeps options set on an aggregate
+
+`Expr.over()` on an aggregate used to drop the `filter`, `distinct`,
+`null_treatment`, and `order_by` options it was built with. The first three are
+now kept, which can change results:
+
+```python
+f.avg(col("v"), distinct=True).over(Window())
+# v = [1, 1, 4]; before: 2.0, after: 2.5
+```
+
+An `order_by` on the aggregate now raises instead of being dropped, as it does
+with `OVER` in SQL. Remove it, or move it into the `Window` if it was meant to
+order the rows:
+
+```python
+# before: order_by dropped; after: raises
+f.first_value(col("v"), order_by=col("i").sort(ascending=False)).over(
+ Window(partition_by=[col("g")])
+)
+
+# after: the Window orders the rows the aggregate sees
+f.first_value(col("v")).over(
+ Window(partition_by=[col("g")], order_by=[col("i").sort(ascending=False)])
+)
+```
+
+A `WITHIN GROUP` function such as `percentile_cont` still accepts
+an ascending `sort_expression`, and raises on a descending one, which used to
+give the ascending result. See {ref}`aggregate_over_options`.
+
+### Python aggregate UDFs reject `DISTINCT`
+
+A Python {py:class}`~datafusion.user_defined.Accumulator` cannot deduplicate its
+input, so a Python aggregate UDF with `DISTINCT` counted every row. It now
+raises instead of returning that result:
+
+```python
+my_sum(col("v")).over(Window()).distinct().build()
+# v = [1, 1, 1, 5]; before: 8.0; after: DISTINCT is not supported ...
+```
+
+When the optimizer rewrites the query to group by the distinct values first,
+as it does for SQL's `SELECT my_sum(DISTINCT v) FROM t`, the query still runs
+and gives the distinct result.
+
+### Percentile functions keep the sort direction
+
+{py:func}`~datafusion.functions.percentile_cont`,
+{py:func}`~datafusion.functions.quantile_cont`,
+{py:func}`~datafusion.functions.approx_percentile_cont`, and
+{py:func}`~datafusion.functions.approx_percentile_cont_with_weight` ignored the
+direction of `sort_expression`, so a descending sort gave the ascending result.
+Used as aggregates, they now match `WITHIN GROUP (ORDER BY ... DESC)` in SQL
+(see {ref}`aggregate_over_options` for their use in a window):
+
+```python
+f.percentile_cont(col("a").sort(ascending=False), 0.25)
+# a = [1, 2, 3, 4, 5]; before: 2.0, after: 4.0
+```
+
+Their generated column names now include the ordering, in the same form as SQL.
+Code that selects the result by its generated name should `alias()` it instead.
+
+```python
+f.percentile_cont(col("a"), 0.25)
+# before: percentile_cont(t.a,Float64(0.25))
+# after: percentile_cont(Float64(0.25)) WITHIN GROUP [t.a ASC NULLS FIRST]
+```
+
+### `fill_null(subset=[])` fills no columns
+
+{py:meth}`~datafusion.dataframe.DataFrame.fill_null` with an empty `subset`
+list used to fill every column, the same as `subset=None`. It now returns the
+DataFrame unchanged, so a subset computed from the schema that matches nothing
+no longer rewrites every column. The new
+{py:meth}`~datafusion.dataframe.DataFrame.fill_nan` behaves the same way.
+
+```python
+df.fill_null(0, subset=[]) # before: fills all columns; after: fills none
+df.fill_null(0) # fills all columns, before and after
+```
+
+### `spark.last_day` renamed its parameter
+
+The parameter of {py:func}`datafusion.functions.spark.last_day` is now named
+`date`, matching `pyspark.sql.functions.last_day`. Positional calls are
+unaffected; update any call passing it by keyword.
+
+```python
+spark.last_day(col=d) # before
+spark.last_day(date=d) # after
+```
+
+### Changes to the `datafusion-python-util` crate
+
+Extension libraries written in Rust usually depend on the
+`datafusion-python-util` crate for the helpers that read these capsules. Two of
+those helpers changed, because the getter they call now takes the session.
+
+`ffi_logical_codec_from_pycapsule` takes a second argument. Pass `Some(session)`
+when importing an object from another library, so its getter receives the
+session it is being installed on. Pass `None` when the object *is* a session and
+you are asking it for what it holds:
+
+```rust
+// Before
+let codec = ffi_logical_codec_from_pycapsule(obj)?;
+
+// After
+let codec = ffi_logical_codec_from_pycapsule(obj, Some(session))?;
+```
+
+`physical_codec_from_pycapsule` has been **removed**. It called
+`__datafusion_physical_extension_codec__` with no arguments, which no longer
+matches the protocol, so against an updated codec it raised a bare `TypeError`
+and against an outdated one it silently produced a codec bound to the wrong
+session. Use `ffi_physical_codec_from_pycapsule`, which passes the session:
+
+```rust
+// Before
+let codec: Arc<dyn PhysicalExtensionCodec> = physical_codec_from_pycapsule(&obj)?;
+
+// After
+let ffi = ffi_physical_codec_from_pycapsule(obj, Some(session))?;
+let codec: Arc<dyn PhysicalExtensionCodec> = (&ffi).into();
+```
+
+`physical_optimizer_rule_from_pycapsule` and `task_context_from_pycapsule` are
+unchanged. Their hooks take no session.
+
+Calling a getter that still has the old signature now raises an `ImportError`
+naming the method, with the original `TypeError` retained as its `__cause__`,
+rather than a bare `TypeError`.
+
## DataFusion 54.0.0
The `Config` class has been removed. It was a standalone wrapper around
diff --git a/_static/check-solid.svg b/_static/check-solid.svg
new file mode 100644
index 00000000..92fad4b5
--- /dev/null
+++ b/_static/check-solid.svg
@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-check" width="44" height="44" viewBox="0 0 24 24" stroke-width="2" stroke="#22863a" fill="none" stroke-linecap="round" stroke-linejoin="round">
+ <path stroke="none" d="M0 0h24v24H0z" fill="none"/>
+ <path d="M5 12l5 5l10 -10" />
+</svg>
diff --git a/_static/clipboard.min.js b/_static/clipboard.min.js
new file mode 100644
index 00000000..54b3c463
--- /dev/null
+++ b/_static/clipboard.min.js
@@ -0,0 +1,7 @@
+/*!
+ * clipboard.js v2.0.8
+ * https://clipboardjs.com/
+ *
+ * Licensed MIT © Zeno Rocha
+ */
+!function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e():"function"==typeof define&&define.amd?define([],e):"object"==typeof exports?exports.ClipboardJS=e():t.ClipboardJS=e()}(this,function(){return n={686:function(t,e,n){"use strict";n.d(e,{default:function(){return o}});var e=n(279),i=n.n(e),e=n(370),u=n.n(e),e=n(817),c=n.n(e);function a(t){try{return document.execCommand(t)}catch(t){return}}var f=function(t){t=c()(t);return a("cut"),t};var l=function(t){var [...]
\ No newline at end of file
diff --git a/_static/copy-button.svg b/_static/copy-button.svg
new file mode 100644
index 00000000..9c074dae
--- /dev/null
+++ b/_static/copy-button.svg
@@ -0,0 +1,5 @@
+<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-copy" width="44" height="44" viewBox="0 0 24 24" stroke-width="1.5" stroke="#000000" fill="none" stroke-linecap="round" stroke-linejoin="round">
+ <path stroke="none" d="M0 0h24v24H0z" fill="none"/>
+ <rect x="8" y="8" width="12" height="12" rx="2" />
+ <path d="M16 8v-2a2 2 0 0 0 -2 -2h-8a2 2 0 0 0 -2 2v8a2 2 0 0 0 2 2h2" />
+</svg>
diff --git a/_static/copybutton.css b/_static/copybutton.css
new file mode 100644
index 00000000..f1916ec7
--- /dev/null
+++ b/_static/copybutton.css
@@ -0,0 +1,94 @@
+/* Copy buttons */
+button.copybtn {
+ position: absolute;
+ display: flex;
+ top: .3em;
+ right: .3em;
+ width: 1.7em;
+ height: 1.7em;
+ opacity: 0;
+ transition: opacity 0.3s, border .3s, background-color .3s;
+ user-select: none;
+ padding: 0;
+ border: none;
+ outline: none;
+ border-radius: 0.4em;
+ /* The colors that GitHub uses */
+ border: #1b1f2426 1px solid;
+ background-color: #f6f8fa;
+ color: #57606a;
+}
+
+button.copybtn.success {
+ border-color: #22863a;
+ color: #22863a;
+}
+
+button.copybtn svg {
+ stroke: currentColor;
+ width: 1.5em;
+ height: 1.5em;
+ padding: 0.1em;
+}
+
+div.highlight {
+ position: relative;
+}
+
+/* Show the copybutton */
+.highlight:hover button.copybtn, button.copybtn.success {
+ opacity: 1;
+}
+
+.highlight button.copybtn:hover {
+ background-color: rgb(235, 235, 235);
+}
+
+.highlight button.copybtn:active {
+ background-color: rgb(187, 187, 187);
+}
+
+/**
+ * A minimal CSS-only tooltip copied from:
+ * https://codepen.io/mildrenben/pen/rVBrpK
+ *
+ * To use, write HTML like the following:
+ *
+ * <p class="o-tooltip--left" data-tooltip="Hey">Short</p>
+ */
+ .o-tooltip--left {
+ position: relative;
+ }
+
+ .o-tooltip--left:after {
+ opacity: 0;
+ visibility: hidden;
+ position: absolute;
+ content: attr(data-tooltip);
+ padding: .2em;
+ font-size: .8em;
+ left: -.2em;
+ background: grey;
+ color: white;
+ white-space: nowrap;
+ z-index: 2;
+ border-radius: 2px;
+ transform: translateX(-102%) translateY(0);
+ transition: opacity 0.2s cubic-bezier(0.64, 0.09, 0.08, 1), transform 0.2s cubic-bezier(0.64, 0.09, 0.08, 1);
+}
+
+.o-tooltip--left:hover:after {
+ display: block;
+ opacity: 1;
+ visibility: visible;
+ transform: translateX(-100%) translateY(0);
+ transition: opacity 0.2s cubic-bezier(0.64, 0.09, 0.08, 1), transform 0.2s cubic-bezier(0.64, 0.09, 0.08, 1);
+ transition-delay: .5s;
+}
+
+/* By default the copy button shouldn't show up when printing a page */
+@media print {
+ button.copybtn {
+ display: none;
+ }
+}
diff --git a/_static/copybutton.js b/_static/copybutton.js
new file mode 100644
index 00000000..2ea7ff3e
--- /dev/null
+++ b/_static/copybutton.js
@@ -0,0 +1,248 @@
+// Localization support
+const messages = {
+ 'en': {
+ 'copy': 'Copy',
+ 'copy_to_clipboard': 'Copy to clipboard',
+ 'copy_success': 'Copied!',
+ 'copy_failure': 'Failed to copy',
+ },
+ 'es' : {
+ 'copy': 'Copiar',
+ 'copy_to_clipboard': 'Copiar al portapapeles',
+ 'copy_success': '¡Copiado!',
+ 'copy_failure': 'Error al copiar',
+ },
+ 'de' : {
+ 'copy': 'Kopieren',
+ 'copy_to_clipboard': 'In die Zwischenablage kopieren',
+ 'copy_success': 'Kopiert!',
+ 'copy_failure': 'Fehler beim Kopieren',
+ },
+ 'fr' : {
+ 'copy': 'Copier',
+ 'copy_to_clipboard': 'Copier dans le presse-papier',
+ 'copy_success': 'Copié !',
+ 'copy_failure': 'Échec de la copie',
+ },
+ 'ru': {
+ 'copy': 'Скопировать',
+ 'copy_to_clipboard': 'Скопировать в буфер',
+ 'copy_success': 'Скопировано!',
+ 'copy_failure': 'Не удалось скопировать',
+ },
+ 'zh-CN': {
+ 'copy': '复制',
+ 'copy_to_clipboard': '复制到剪贴板',
+ 'copy_success': '复制成功!',
+ 'copy_failure': '复制失败',
+ },
+ 'it' : {
+ 'copy': 'Copiare',
+ 'copy_to_clipboard': 'Copiato negli appunti',
+ 'copy_success': 'Copiato!',
+ 'copy_failure': 'Errore durante la copia',
+ }
+}
+
+let locale = 'en'
+if( document.documentElement.lang !== undefined
+ && messages[document.documentElement.lang] !== undefined ) {
+ locale = document.documentElement.lang
+}
+
+let doc_url_root = DOCUMENTATION_OPTIONS.URL_ROOT;
+if (doc_url_root == '#') {
+ doc_url_root = '';
+}
+
+/**
+ * SVG files for our copy buttons
+ */
+let iconCheck = `<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-check" width="44" height="44" viewBox="0 0 24 24" stroke-width="2" stroke="#22863a" fill="none" stroke-linecap="round" stroke-linejoin="round">
+ <title>${messages[locale]['copy_success']}</title>
+ <path stroke="none" d="M0 0h24v24H0z" fill="none"/>
+ <path d="M5 12l5 5l10 -10" />
+</svg>`
+
+// If the user specified their own SVG use that, otherwise use the default
+let iconCopy = ``;
+if (!iconCopy) {
+ iconCopy = `<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-copy" width="44" height="44" viewBox="0 0 24 24" stroke-width="1.5" stroke="#000000" fill="none" stroke-linecap="round" stroke-linejoin="round">
+ <title>${messages[locale]['copy_to_clipboard']}</title>
+ <path stroke="none" d="M0 0h24v24H0z" fill="none"/>
+ <rect x="8" y="8" width="12" height="12" rx="2" />
+ <path d="M16 8v-2a2 2 0 0 0 -2 -2h-8a2 2 0 0 0 -2 2v8a2 2 0 0 0 2 2h2" />
+</svg>`
+}
+
+/**
+ * Set up copy/paste for code blocks
+ */
+
+const runWhenDOMLoaded = cb => {
+ if (document.readyState != 'loading') {
+ cb()
+ } else if (document.addEventListener) {
+ document.addEventListener('DOMContentLoaded', cb)
+ } else {
+ document.attachEvent('onreadystatechange', function() {
+ if (document.readyState == 'complete') cb()
+ })
+ }
+}
+
+const codeCellId = index => `codecell${index}`
+
+// Clears selected text since ClipboardJS will select the text when copying
+const clearSelection = () => {
+ if (window.getSelection) {
+ window.getSelection().removeAllRanges()
+ } else if (document.selection) {
+ document.selection.empty()
+ }
+}
+
+// Changes tooltip text for a moment, then changes it back
+// We want the timeout of our `success` class to be a bit shorter than the
+// tooltip and icon change, so that we can hide the icon before changing back.
+var timeoutIcon = 2000;
+var timeoutSuccessClass = 1500;
+
+const temporarilyChangeTooltip = (el, oldText, newText) => {
+ el.setAttribute('data-tooltip', newText)
+ el.classList.add('success')
+ // Remove success a little bit sooner than we change the tooltip
+ // So that we can use CSS to hide the copybutton first
+ setTimeout(() => el.classList.remove('success'), timeoutSuccessClass)
+ setTimeout(() => el.setAttribute('data-tooltip', oldText), timeoutIcon)
+}
+
+// Changes the copy button icon for two seconds, then changes it back
+const temporarilyChangeIcon = (el) => {
+ el.innerHTML = iconCheck;
+ setTimeout(() => {el.innerHTML = iconCopy}, timeoutIcon)
+}
+
+const addCopyButtonToCodeCells = () => {
+ // If ClipboardJS hasn't loaded, wait a bit and try again. This
+ // happens because we load ClipboardJS asynchronously.
+ if (window.ClipboardJS === undefined) {
+ setTimeout(addCopyButtonToCodeCells, 250)
+ return
+ }
+
+ // Add copybuttons to all of our code cells
+ const COPYBUTTON_SELECTOR = 'div.highlight pre';
+ const codeCells = document.querySelectorAll(COPYBUTTON_SELECTOR)
+ codeCells.forEach((codeCell, index) => {
+ const id = codeCellId(index)
+ codeCell.setAttribute('id', id)
+
+ const clipboardButton = id =>
+ `<button class="copybtn o-tooltip--left" data-tooltip="${messages[locale]['copy']}" data-clipboard-target="#${id}">
+ ${iconCopy}
+ </button>`
+ codeCell.insertAdjacentHTML('afterend', clipboardButton(id))
+ })
+
+function escapeRegExp(string) {
+ return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
+}
+
+/**
+ * Removes excluded text from a Node.
+ *
+ * @param {Node} target Node to filter.
+ * @param {string} exclude CSS selector of nodes to exclude.
+ * @returns {DOMString} Text from `target` with text removed.
+ */
+function filterText(target, exclude) {
+ const clone = target.cloneNode(true); // clone as to not modify the live DOM
+ if (exclude) {
+ // remove excluded nodes
+ clone.querySelectorAll(exclude).forEach(node => node.remove());
+ }
+ return clone.innerText;
+}
+
+// Callback when a copy button is clicked. Will be passed the node that was clicked
+// should then grab the text and replace pieces of text that shouldn't be used in output
+function formatCopyText(textContent, copybuttonPromptText, isRegexp = false, onlyCopyPromptLines = true, removePrompts = true, copyEmptyLines = true, lineContinuationChar = "", hereDocDelim = "") {
+ var regexp;
+ var match;
+
+ // Do we check for line continuation characters and "HERE-documents"?
+ var useLineCont = !!lineContinuationChar
+ var useHereDoc = !!hereDocDelim
+
+ // create regexp to capture prompt and remaining line
+ if (isRegexp) {
+ regexp = new RegExp('^(' + copybuttonPromptText + ')(.*)')
+ } else {
+ regexp = new RegExp('^(' + escapeRegExp(copybuttonPromptText) + ')(.*)')
+ }
+
+ const outputLines = [];
+ var promptFound = false;
+ var gotLineCont = false;
+ var gotHereDoc = false;
+ const lineGotPrompt = [];
+ for (const line of textContent.split('\n')) {
+ match = line.match(regexp)
+ if (match || gotLineCont || gotHereDoc) {
+ promptFound = regexp.test(line)
+ lineGotPrompt.push(promptFound)
+ if (removePrompts && promptFound) {
+ outputLines.push(match[2])
+ } else {
+ outputLines.push(line)
+ }
+ gotLineCont = line.endsWith(lineContinuationChar) & useLineCont
+ if (line.includes(hereDocDelim) & useHereDoc)
+ gotHereDoc = !gotHereDoc
+ } else if (!onlyCopyPromptLines) {
+ outputLines.push(line)
+ } else if (copyEmptyLines && line.trim() === '') {
+ outputLines.push(line)
+ }
+ }
+
+ // If no lines with the prompt were found then just use original lines
+ if (lineGotPrompt.some(v => v === true)) {
+ textContent = outputLines.join('\n');
+ }
+
+ // Remove a trailing newline to avoid auto-running when pasting
+ if (textContent.endsWith("\n")) {
+ textContent = textContent.slice(0, -1)
+ }
+ return textContent
+}
+
+
+var copyTargetText = (trigger) => {
+ var target = document.querySelector(trigger.attributes['data-clipboard-target'].value);
+
+ // get filtered text
+ let exclude = '.linenos';
+
+ let text = filterText(target, exclude);
+ return formatCopyText(text, '', false, true, true, true, '', '')
+}
+
+ // Initialize with a callback so we can modify the text before copy
+ const clipboard = new ClipboardJS('.copybtn', {text: copyTargetText})
+
+ // Update UI with error/success messages
+ clipboard.on('success', event => {
+ clearSelection()
+ temporarilyChangeTooltip(event.trigger, messages[locale]['copy'], messages[locale]['copy_success'])
+ temporarilyChangeIcon(event.trigger)
+ })
+
+ clipboard.on('error', event => {
+ temporarilyChangeTooltip(event.trigger, messages[locale]['copy'], messages[locale]['copy_failure'])
+ })
+}
+
+runWhenDOMLoaded(addCopyButtonToCodeCells)
\ No newline at end of file
diff --git a/_static/copybutton_funcs.js b/_static/copybutton_funcs.js
new file mode 100644
index 00000000..dbe1aaad
--- /dev/null
+++ b/_static/copybutton_funcs.js
@@ -0,0 +1,73 @@
+function escapeRegExp(string) {
+ return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
+}
+
+/**
+ * Removes excluded text from a Node.
+ *
+ * @param {Node} target Node to filter.
+ * @param {string} exclude CSS selector of nodes to exclude.
+ * @returns {DOMString} Text from `target` with text removed.
+ */
+export function filterText(target, exclude) {
+ const clone = target.cloneNode(true); // clone as to not modify the live DOM
+ if (exclude) {
+ // remove excluded nodes
+ clone.querySelectorAll(exclude).forEach(node => node.remove());
+ }
+ return clone.innerText;
+}
+
+// Callback when a copy button is clicked. Will be passed the node that was clicked
+// should then grab the text and replace pieces of text that shouldn't be used in output
+export function formatCopyText(textContent, copybuttonPromptText, isRegexp = false, onlyCopyPromptLines = true, removePrompts = true, copyEmptyLines = true, lineContinuationChar = "", hereDocDelim = "") {
+ var regexp;
+ var match;
+
+ // Do we check for line continuation characters and "HERE-documents"?
+ var useLineCont = !!lineContinuationChar
+ var useHereDoc = !!hereDocDelim
+
+ // create regexp to capture prompt and remaining line
+ if (isRegexp) {
+ regexp = new RegExp('^(' + copybuttonPromptText + ')(.*)')
+ } else {
+ regexp = new RegExp('^(' + escapeRegExp(copybuttonPromptText) + ')(.*)')
+ }
+
+ const outputLines = [];
+ var promptFound = false;
+ var gotLineCont = false;
+ var gotHereDoc = false;
+ const lineGotPrompt = [];
+ for (const line of textContent.split('\n')) {
+ match = line.match(regexp)
+ if (match || gotLineCont || gotHereDoc) {
+ promptFound = regexp.test(line)
+ lineGotPrompt.push(promptFound)
+ if (removePrompts && promptFound) {
+ outputLines.push(match[2])
+ } else {
+ outputLines.push(line)
+ }
+ gotLineCont = line.endsWith(lineContinuationChar) & useLineCont
+ if (line.includes(hereDocDelim) & useHereDoc)
+ gotHereDoc = !gotHereDoc
+ } else if (!onlyCopyPromptLines) {
+ outputLines.push(line)
+ } else if (copyEmptyLines && line.trim() === '') {
+ outputLines.push(line)
+ }
+ }
+
+ // If no lines with the prompt were found then just use original lines
+ if (lineGotPrompt.some(v => v === true)) {
+ textContent = outputLines.join('\n');
+ }
+
+ // Remove a trailing newline to avoid auto-running when pasting
+ if (textContent.endsWith("\n")) {
+ textContent = textContent.slice(0, -1)
+ }
+ return textContent
+}
diff --git a/autoapi/datafusion/catalog/index.html b/autoapi/datafusion/catalog/index.html
index a27e497c..d2d6a2ef 100644
--- a/autoapi/datafusion/catalog/index.html
+++ b/autoapi/datafusion/catalog/index.html
@@ -33,6 +33,7 @@
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=8f2a1f02" />
<link rel="stylesheet" type="text/css" href="../../../_static/mystnb.11b39860a7a0cbfd473a3ad8a317855267ff0bd372690045ca344a6b62be495e.css" />
<link rel="stylesheet" type="text/css" href="../../../_static/graphviz.css?v=4ae1632d" />
+ <link rel="stylesheet" type="text/css" href="../../../_static/copybutton.css?v=76b2166b" />
<link rel="stylesheet" type="text/css" href="../../../_static/theme_overrides.css?v=4af573bc" />
<!-- So that users can add custom icons -->
@@ -44,6 +45,8 @@
<script src="../../../_static/documentation_options.js?v=5929fcd5"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
+ <script src="../../../_static/clipboard.min.js?v=a7894cd8"></script>
+ <script src="../../../_static/copybutton.js?v=f281be69"></script>
<script>DOCUMENTATION_OPTIONS.pagename = 'autoapi/datafusion/catalog/index';</script>
<script src="../../../_static/toc-toggle.js?v=36375f01"></script>
<link rel="icon" href="../../../_static/favicon.svg"/>
@@ -142,6 +145,13 @@
</li>
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../../extension-guide/index.html">
+ Extension Guide
+ </a>
+</li>
+
+
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../../contributor-guide/index.html">
Contributor Guide
@@ -270,6 +280,13 @@
</li>
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../../extension-guide/index.html">
+ Extension Guide
+ </a>
+</li>
+
+
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../../contributor-guide/index.html">
Contributor Guide
@@ -380,15 +397,34 @@
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal" href="../../../user-guide/configuration.html">Configuration</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../../../user-guide/distributing-work.html">Distributing work</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../user-guide/extensions.html">Using extension libraries</a></li>
+<li class="toctree-l2 has-children"><a class="reference internal" href="../../../user-guide/distributing-work/index.html">Distributing work</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/distributing-work/expressions.html">Shipping expressions to workers</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/distributing-work/query-engines.html">Distributed query engines</a></li>
+</ul>
+</details></li>
<li class="toctree-l2"><a class="reference internal" href="../../../user-guide/sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../user-guide/upgrade-guides.html">Upgrade Guides</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../user-guide/ai-coding-assistants.html">Using AI Coding Assistants</a></li>
</ul>
</details></li>
+<li class="toctree-l1 has-children"><a class="reference internal" href="../../../extension-guide/index.html">Extension Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/why-ffi.html">Why FFI</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/capsule-protocol.html">The capsule protocol</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/table-providers.html">Providers and catalogs</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/functions.html">Functions and table functions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/codecs.html">Extension codecs</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/bundles.html">Extension bundles</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/query-planners.html">Query planners</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/other-components.html">Optimizer rules and configuration</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/sessions.html">Sessions, handles, and lifetimes</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/checklist.html">Extension author checklist</a></li>
+</ul>
+</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../../../contributor-guide/index.html">Contributor Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../../../contributor-guide/introduction.html">Introduction</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../../../contributor-guide/ffi.html">Python Extensions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../contributor-guide/pyo3-guidelines.html">PyO3 binding guidelines</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../contributor-guide/ffi-internals.html">FFI framing internals</a></li>
</ul>
</details></li>
<li class="toctree-l1 current active has-children"><a class="reference internal" href="../../index.html">API Reference</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current">
@@ -398,6 +434,7 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../extensions/index.html">datafusion.extensions</a></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
</ul>
@@ -518,6 +555,9 @@
<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.catalog.Table" title="datafusion.catalog.Table"><code class="xref py py-obj docutils literal notranslate"><span class="pre">Table</span></code></a></p></td>
<td><p>A DataFusion table.</p></td>
</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.catalog.TableProviderFactory" title="datafusion.catalog.TableProviderFactory"><code class="xref py py-obj docutils literal notranslate"><span class="pre">TableProviderFactory</span></code></a></p></td>
+<td><p>Abstract class for defining a Python based Table Provider Factory.</p></td>
+</tr>
</tbody>
</table>
</div>
@@ -854,6 +894,19 @@ interface (advanced)</p></li>
</dd></dl>
+<dl class="py class">
+<dt class="sig sig-object py" id="datafusion.catalog.TableProviderFactory">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.catalog.</span></span><span class="sig-name descname"><span class="pre">TableProviderFactory</span></span><a class="headerlink" href="#datafusion.catalog.TableProviderFactory" title="Link to this definition">#</a></dt>
+<dd><p>Bases: <code class="xref py py-obj docutils literal notranslate"><span class="pre">abc.ABC</span></code></p>
+<p>Abstract class for defining a Python based Table Provider Factory.</p>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.catalog.TableProviderFactory.create">
+<em class="property"><span class="pre">abstract</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">create</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">cmd</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datafusion.expr.CreateExternalTable</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="si [...]
+<dd><p>Create a table using the <code class="xref py py-class docutils literal notranslate"><span class="pre">CreateExternalTable</span></code>.</p>
+</dd></dl>
+
+</dd></dl>
+
</section>
</section>
@@ -970,6 +1023,10 @@ interface (advanced)</p></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.catalog.Table.schema"><code class="docutils literal notranslate"><span class="pre">Table.schema</span></code></a></li>
</ul>
</li>
+<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.catalog.TableProviderFactory"><code class="docutils literal notranslate"><span class="pre">TableProviderFactory</span></code></a><ul class="nav section-nav flex-column">
+<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#datafusion.catalog.TableProviderFactory.create"><code class="docutils literal notranslate"><span class="pre">TableProviderFactory.create()</span></code></a></li>
+</ul>
+</li>
</ul>
</li>
</ul>
diff --git a/autoapi/datafusion/context/index.html b/autoapi/datafusion/context/index.html
index 38f7297a..1e223b6d 100644
--- a/autoapi/datafusion/context/index.html
+++ b/autoapi/datafusion/context/index.html
@@ -33,6 +33,7 @@
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=8f2a1f02" />
<link rel="stylesheet" type="text/css" href="../../../_static/mystnb.11b39860a7a0cbfd473a3ad8a317855267ff0bd372690045ca344a6b62be495e.css" />
<link rel="stylesheet" type="text/css" href="../../../_static/graphviz.css?v=4ae1632d" />
+ <link rel="stylesheet" type="text/css" href="../../../_static/copybutton.css?v=76b2166b" />
<link rel="stylesheet" type="text/css" href="../../../_static/theme_overrides.css?v=4af573bc" />
<!-- So that users can add custom icons -->
@@ -44,6 +45,8 @@
<script src="../../../_static/documentation_options.js?v=5929fcd5"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
+ <script src="../../../_static/clipboard.min.js?v=a7894cd8"></script>
+ <script src="../../../_static/copybutton.js?v=f281be69"></script>
<script>DOCUMENTATION_OPTIONS.pagename = 'autoapi/datafusion/context/index';</script>
<script src="../../../_static/toc-toggle.js?v=36375f01"></script>
<link rel="icon" href="../../../_static/favicon.svg"/>
@@ -142,6 +145,13 @@
</li>
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../../extension-guide/index.html">
+ Extension Guide
+ </a>
+</li>
+
+
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../../contributor-guide/index.html">
Contributor Guide
@@ -270,6 +280,13 @@
</li>
+<li class="nav-item ">
+ <a class="nav-link nav-internal" href="../../../extension-guide/index.html">
+ Extension Guide
+ </a>
+</li>
+
+
<li class="nav-item ">
<a class="nav-link nav-internal" href="../../../contributor-guide/index.html">
Contributor Guide
@@ -380,15 +397,34 @@
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal" href="../../../user-guide/configuration.html">Configuration</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../../../user-guide/distributing-work.html">Distributing work</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../user-guide/extensions.html">Using extension libraries</a></li>
+<li class="toctree-l2 has-children"><a class="reference internal" href="../../../user-guide/distributing-work/index.html">Distributing work</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/distributing-work/expressions.html">Shipping expressions to workers</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../../../user-guide/distributing-work/query-engines.html">Distributed query engines</a></li>
+</ul>
+</details></li>
<li class="toctree-l2"><a class="reference internal" href="../../../user-guide/sql.html">SQL</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../user-guide/upgrade-guides.html">Upgrade Guides</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../user-guide/ai-coding-assistants.html">Using AI Coding Assistants</a></li>
</ul>
</details></li>
+<li class="toctree-l1 has-children"><a class="reference internal" href="../../../extension-guide/index.html">Extension Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/why-ffi.html">Why FFI</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/capsule-protocol.html">The capsule protocol</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/table-providers.html">Providers and catalogs</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/functions.html">Functions and table functions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/codecs.html">Extension codecs</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/bundles.html">Extension bundles</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/query-planners.html">Query planners</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/other-components.html">Optimizer rules and configuration</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/sessions.html">Sessions, handles, and lifetimes</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../extension-guide/checklist.html">Extension author checklist</a></li>
+</ul>
+</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../../../contributor-guide/index.html">Contributor Guide</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../../../contributor-guide/introduction.html">Introduction</a></li>
-<li class="toctree-l2"><a class="reference internal" href="../../../contributor-guide/ffi.html">Python Extensions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../contributor-guide/pyo3-guidelines.html">PyO3 binding guidelines</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../contributor-guide/ffi-internals.html">FFI framing internals</a></li>
</ul>
</details></li>
<li class="toctree-l1 current active has-children"><a class="reference internal" href="../../index.html">API Reference</a><details open="open"><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul class="current">
@@ -398,6 +434,7 @@
<li class="toctree-l3"><a class="reference internal" href="../dataframe/index.html">datafusion.dataframe</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dataframe_formatter/index.html">datafusion.dataframe_formatter</a></li>
<li class="toctree-l3"><a class="reference internal" href="../expr/index.html">datafusion.expr</a></li>
+<li class="toctree-l3"><a class="reference internal" href="../extensions/index.html">datafusion.extensions</a></li>
<li class="toctree-l3 has-children"><a class="reference internal" href="../functions/index.html">datafusion.functions</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l4"><a class="reference internal" href="../functions/spark/index.html">datafusion.functions.spark</a></li>
</ul>
@@ -525,22 +562,19 @@ execution model.</p>
<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.context.ArrowStreamExportable" title="datafusion.context.ArrowStreamExportable"><code class="xref py py-obj docutils literal notranslate"><span class="pre">ArrowStreamExportable</span></code></a></p></td>
<td><p>Type hint for object exporting Arrow C Stream via Arrow PyCapsule Interface.</p></td>
</tr>
-<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.context.PhysicalOptimizerRuleExportable" title="datafusion.context.PhysicalOptimizerRuleExportable"><code class="xref py py-obj docutils literal notranslate"><span class="pre">PhysicalOptimizerRuleExportable</span></code></a></p></td>
-<td><p>Type hint for object that has __datafusion_physical_optimizer_rule__ PyCapsule.</p></td>
-</tr>
-<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.context.RuntimeEnvBuilder" title="datafusion.context.RuntimeEnvBuilder"><code class="xref py py-obj docutils literal notranslate"><span class="pre">RuntimeEnvBuilder</span></code></a></p></td>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.context.RuntimeEnvBuilder" title="datafusion.context.RuntimeEnvBuilder"><code class="xref py py-obj docutils literal notranslate"><span class="pre">RuntimeEnvBuilder</span></code></a></p></td>
<td><p>Runtime configuration options.</p></td>
</tr>
-<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.context.SQLOptions" title="datafusion.context.SQLOptions"><code class="xref py py-obj docutils literal notranslate"><span class="pre">SQLOptions</span></code></a></p></td>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.context.SQLOptions" title="datafusion.context.SQLOptions"><code class="xref py py-obj docutils literal notranslate"><span class="pre">SQLOptions</span></code></a></p></td>
<td><p>Options to be used when performing SQL queries.</p></td>
</tr>
-<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.context.SessionConfig" title="datafusion.context.SessionConfig"><code class="xref py py-obj docutils literal notranslate"><span class="pre">SessionConfig</span></code></a></p></td>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.context.SessionConfig" title="datafusion.context.SessionConfig"><code class="xref py py-obj docutils literal notranslate"><span class="pre">SessionConfig</span></code></a></p></td>
<td><p>Session configuration options.</p></td>
</tr>
-<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-obj docutils literal notranslate"><span class="pre">SessionContext</span></code></a></p></td>
+<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-obj docutils literal notranslate"><span class="pre">SessionContext</span></code></a></p></td>
<td><p>This is the main interface for executing queries and creating DataFrames.</p></td>
</tr>
-<tr class="row-even"><td><p><a class="reference internal" href="#datafusion.context.TableProviderExportable" title="datafusion.context.TableProviderExportable"><code class="xref py py-obj docutils literal notranslate"><span class="pre">TableProviderExportable</span></code></a></p></td>
+<tr class="row-odd"><td><p><a class="reference internal" href="#datafusion.context.TableProviderExportable" title="datafusion.context.TableProviderExportable"><code class="xref py py-obj docutils literal notranslate"><span class="pre">TableProviderExportable</span></code></a></p></td>
<td><p>Type hint for object that has __datafusion_table_provider__ PyCapsule.</p></td>
</tr>
</tbody>
@@ -575,20 +609,6 @@ execution model.</p>
</dd></dl>
-<dl class="py class">
-<dt class="sig sig-object py" id="datafusion.context.PhysicalOptimizerRuleExportable">
-<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.context.</span></span><span class="sig-name descname"><span class="pre">PhysicalOptimizerRuleExportable</span></span><a class="headerlink" href="#datafusion.context.PhysicalOptimizerRuleExportable" title="Link to this definition">#</a></dt>
-<dd><p>Bases: <code class="xref py py-obj docutils literal notranslate"><span class="pre">Protocol</span></code></p>
-<p>Type hint for object that has __datafusion_physical_optimizer_rule__ PyCapsule.</p>
-<p>The method returns a PyCapsule wrapping an <code class="docutils literal notranslate"><span class="pre">FFI_PhysicalOptimizerRule</span></code>,
-typically produced by a separate compiled extension.</p>
-<dl class="py method">
-<dt class="sig sig-object py" id="datafusion.context.PhysicalOptimizerRuleExportable.__datafusion_physical_optimizer_rule__">
-<span class="sig-name descname"><span class="pre">__datafusion_physical_optimizer_rule__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">object</span></span></span><a class="headerlink" href="#datafusion.context.PhysicalOptimizerRuleExportable.__datafusion_physical_optimizer_rule__" title="Link to this definition">#</a></dt>
-<dd></dd></dl>
-
-</dd></dl>
-
<dl class="py class">
<dt class="sig sig-object py" id="datafusion.context.RuntimeEnvBuilder">
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.context.</span></span><span class="sig-name descname"><span class="pre">RuntimeEnvBuilder</span></span><a class="headerlink" href="#datafusion.context.RuntimeEnvBuilder" title="Link to this definition">#</a></dt>
@@ -799,23 +819,69 @@ multiple spillable operators that all will spill.</p>
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.context.</span></span><span class="sig-name descname"><span class="pre">SessionConfig</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">config_options</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">dict</span><span clas [...]
<dd><p>Session configuration options.</p>
<p>Create a new <a class="reference internal" href="#datafusion.context.SessionConfig" title="datafusion.context.SessionConfig"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionConfig</span></code></a> with the given configuration options.</p>
+<p>Each entry is applied as though passed to <a class="reference internal" href="#datafusion.context.SessionConfig.set" title="datafusion.context.SessionConfig.set"><code class="xref py py-meth docutils literal notranslate"><span class="pre">set()</span></code></a>, so the same
+keys are rejected. See <a class="reference internal" href="../../../user-guide/configuration.html#configuration"><span class="std std-ref">Configuration</span></a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
-<dd class="field-odd"><p><strong>config_options</strong> – Configuration options.</p>
+<dd class="field-odd"><p><strong>config_options</strong> – Options to apply, keyed by fully qualified name.</p>
+</dd>
+<dt class="field-even">Raises<span class="colon">:</span></dt>
+<dd class="field-even"><p><strong>ValueError</strong> – If a key names no known option, or a value does not
+parse as that option’s declared type. Which of several bad
+entries is reported is not defined.</p>
</dd>
</dl>
+<p>Example usage:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionConfig</span><span class="p">,</span> <span class="n">SessionContext</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">(</span><span class="n">SessionConfig</span><span class="p">())</span>
+<span class="gp">>>> </span><span class="n">config</span> <span class="o">=</span> <span class="n">SessionConfig</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">config_options</span><span class="o">=</span><span class="p">{</span><span class="s2">"datafusion.execution.batch_size"</span><span class="p">:</span> <span class="s2">"1024"</span><span class="p">}</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">with_information_schema</span><span class="p">(</span><span class="kc">True</span><span class="p">))</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span>
+<span class="gp">... </span> <span class="s2">"select value from information_schema.df_settings"</span>
+<span class="gp">... </span> <span class="s2">" where name = 'datafusion.execution.batch_size'"</span>
+<span class="gp">... </span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()[</span><span class="mi">0</span><span class="p">][</span><span class="s2">"value"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
+<span class="go"><pyarrow.StringScalar: '1024'></span>
+</pre></div>
+</div>
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionConfig.set">
<span class="sig-name descname"><span class="pre">set</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">key</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">value</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</sp [...]
-<dd><p>Set a configuration option.</p>
-<p>Args:
-key: Option key.
-value: Option value.</p>
+<dd><p>Set a configuration option by its fully qualified key.</p>
+<p>Not every key that <code class="docutils literal notranslate"><span class="pre">information_schema.df_settings</span></code> lists can be set
+here: the <code class="docutils literal notranslate"><span class="pre">datafusion.runtime.*</span></code> entries come from the runtime
+environment rather than from the session config. See
+<a class="reference internal" href="../../../user-guide/configuration.html#configuration"><span class="std std-ref">Configuration</span></a>.</p>
<dl class="field-list simple">
-<dt class="field-odd">Returns<span class="colon">:</span></dt>
-<dd class="field-odd"><p>A new <a class="reference internal" href="#datafusion.context.SessionConfig" title="datafusion.context.SessionConfig"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionConfig</span></code></a> object with the updated setting.</p>
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>key</strong> – Option key including its namespace, such as
+<code class="docutils literal notranslate"><span class="pre">datafusion.execution.batch_size</span></code>.</p></li>
+<li><p><strong>value</strong> – Option value as a string, parsed according to the type the
+option declares.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>This <a class="reference internal" href="#datafusion.context.SessionConfig" title="datafusion.context.SessionConfig"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionConfig</span></code></a>, modified in place, so that calls
+chain.</p>
+</dd>
+<dt class="field-odd">Raises<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>ValueError</strong> – If <code class="docutils literal notranslate"><span class="pre">key</span></code> names no known option, or if <code class="docutils literal notranslate"><span class="pre">value</span></code> does
+not parse as that option’s declared type.</p>
</dd>
</dl>
+<p>Example usage:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionConfig</span><span class="p">,</span> <span class="n">SessionContext</span>
+<span class="gp">>>> </span><span class="n">config</span> <span class="o">=</span> <span class="n">SessionConfig</span><span class="p">()</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s2">"datafusion.execution.batch_size"</span><span class="p">,</span> <span class="s2">"1024"</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">with_information_schema</span><span class="p">(</span><span class="kc">True</span><span class="p">))</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span>
+<span class="gp">... </span> <span class="s2">"select value from information_schema.df_settings"</span>
+<span class="gp">... </span> <span class="s2">" where name = 'datafusion.execution.batch_size'"</span>
+<span class="gp">... </span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()[</span><span class="mi">0</span><span class="p">][</span><span class="s2">"value"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
+<span class="go"><pyarrow.StringScalar: '1024'></span>
+</pre></div>
+</div>
</dd></dl>
<dl class="py method">
@@ -1024,6 +1090,24 @@ automatically created.</p>
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.context.</span></span><span class="sig-name descname"><span class="pre">SessionContext</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">config</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusio [...]
<dd><p>This is the main interface for executing queries and creating DataFrames.</p>
<p>See <a class="reference internal" href="../../../user-guide/basics.html#user-guide-concepts"><span class="std std-ref">Concepts</span></a> in the online documentation for more information.</p>
+<p><strong>A context is a handle on a session, not the session itself.</strong> The
+<code class="docutils literal notranslate"><span class="pre">with_*</span></code> methods — <a class="reference internal" href="#datafusion.context.SessionContext.with_logical_extension_codec" title="datafusion.context.SessionContext.with_logical_extension_codec"><code class="xref py py-meth docutils literal notranslate"><span class="pre">with_logical_extension_codec()</span></code></a>,
+<a class="reference internal" href="#datafusion.context.SessionContext.with_physical_extension_codec" title="datafusion.context.SessionContext.with_physical_extension_codec"><code class="xref py py-meth docutils literal notranslate"><span class="pre">with_physical_extension_codec()</span></code></a>,
+<a class="reference internal" href="#datafusion.context.SessionContext.with_python_udf_inlining" title="datafusion.context.SessionContext.with_python_udf_inlining"><code class="xref py py-meth docutils literal notranslate"><span class="pre">with_python_udf_inlining()</span></code></a>, and <a class="reference internal" href="#datafusion.context.SessionContext.with_extensions" title="datafusion.context.SessionContext.with_extensions"><code class="xref py py-meth docutils literal notransla [...]
+return a new context wrapping the <em>same</em> underlying session. Only the
+Python-side codec settings differ; catalogs, tables, registered functions,
+and configuration are the one shared session, so a registration through
+either handle is visible to both.</p>
+<p>A few things therefore belong to the session rather than to a handle, and
+take effect even if the handle that set them is discarded: the query
+planner (see <a class="reference internal" href="#datafusion.context.SessionContext.set_query_planner" title="datafusion.context.SessionContext.set_query_planner"><code class="xref py py-meth docutils literal notranslate"><span class="pre">set_query_planner()</span></code></a>), and the rebuild of an installed
+foreign planner that follows installing a codec. <a class="reference internal" href="../../../extension-guide/sessions.html#extension-sessions"><span class="std std-ref">Sessions, handles, and lifetimes</span></a>
+in the online documentation works through when that matters.</p>
+<p><strong>Keep a context alive for as long as anything derived from it is in use.</strong>
+A <code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code>, logical plan, or exported capsule does
+not extend the session’s lifetime. Once the last context on a session is
+collected, any operation that reaches an extension codec fails with
+<code class="docutils literal notranslate"><span class="pre">TaskContextProvider</span> <span class="pre">went</span> <span class="pre">out</span> <span class="pre">of</span> <span class="pre">scope</span> <span class="pre">over</span> <span class="pre">FFI</span> <span class="pre">boundary</span></code>.</p>
<p>Main interface for executing queries with DataFusion.</p>
<p>Maintains the state of the connection between a user and an instance
of the connection between a user and an instance of the DataFusion
@@ -1047,14 +1131,82 @@ a query against a CSV data source using the <code class="xref py py-class docuti
</div>
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.__datafusion_logical_extension_codec__">
-<span class="sig-name descname"><span class="pre">__datafusion_logical_extension_codec__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">Any</span></span></span><a class="headerlink" href="#datafusion.context.SessionContext.__datafusion_logical_extension_codec__" title="Link to this definition">#</a></dt>
+<span class="sig-name descname"><span class="pre">__datafusion_logical_extension_codec__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">session</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></s [...]
<dd><p>Access the PyCapsule FFI_LogicalExtensionCodec.</p>
+<p><code class="docutils literal notranslate"><span class="pre">session</span></code> is accepted so a context satisfies the same protocol an
+extension library implements, where the argument is how the library
+reaches the host’s codec. A context already carries one, so the
+argument is ignored. See
+<a class="reference internal" href="../user_defined/index.html#datafusion.user_defined.LogicalExtensionCodecExportable" title="datafusion.user_defined.LogicalExtensionCodecExportable"><code class="xref py py-class docutils literal notranslate"><span class="pre">LogicalExtensionCodecExportable</span></code></a>
+for what an extension library does with it.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>session</strong> – Accepted and ignored.</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>A <code class="docutils literal notranslate"><span class="pre">datafusion_logical_extension_codec</span></code> PyCapsule.</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span>
+<span class="gp">>>> </span><span class="nb">type</span><span class="p">(</span><span class="n">SessionContext</span><span class="p">()</span><span class="o">.</span><span class="n">__datafusion_logical_extension_codec__</span><span class="p">())</span><span class="o">.</span><span class="vm">__name__</span>
+<span class="go">'PyCapsule'</span>
+</pre></div>
+</div>
</dd></dl>
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.__datafusion_physical_extension_codec__">
-<span class="sig-name descname"><span class="pre">__datafusion_physical_extension_codec__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">Any</span></span></span><a class="headerlink" href="#datafusion.context.SessionContext.__datafusion_physical_extension_codec__" title="Link to this definition">#</a></dt>
+<span class="sig-name descname"><span class="pre">__datafusion_physical_extension_codec__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">session</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></ [...]
<dd><p>Access the PyCapsule FFI_PhysicalExtensionCodec.</p>
+<p>See <a class="reference internal" href="#datafusion.context.SessionContext.__datafusion_logical_extension_codec__" title="datafusion.context.SessionContext.__datafusion_logical_extension_codec__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__datafusion_logical_extension_codec__()</span></code></a> for <code class="docutils literal notranslate"><span class="pre">session</span></code>.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>session</strong> – Accepted and ignored.</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>A <code class="docutils literal notranslate"><span class="pre">datafusion_physical_extension_codec</span></code> PyCapsule.</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="nb">type</span><span class="p">(</span><span class="n">ctx</span><span class="o">.</span><span class="n">__datafusion_physical_extension_codec__</span><span class="p">())</span><span class="o">.</span><span class="vm">__name__</span>
+<span class="go">'PyCapsule'</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.__datafusion_query_planner__">
+<span class="sig-name descname"><span class="pre">__datafusion_query_planner__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">session</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Any</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>< [...]
+<dd><p>Access the <code class="docutils literal notranslate"><span class="pre">FFI_QueryPlanner</span></code> PyCapsule for the current planner.</p>
+<p>This is how you capture the planner a session already has in order to
+wrap it. Capture it <em>before</em> installing the new one, since
+<a class="reference internal" href="#datafusion.context.SessionContext.set_query_planner" title="datafusion.context.SessionContext.set_query_planner"><code class="xref py py-meth docutils literal notranslate"><span class="pre">set_query_planner()</span></code></a> replaces rather than layers.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>session</strong> – Accepted and ignored. See
+<a class="reference internal" href="#datafusion.context.SessionContext.__datafusion_logical_extension_codec__" title="datafusion.context.SessionContext.__datafusion_logical_extension_codec__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__datafusion_logical_extension_codec__()</span></code></a>.</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>A <code class="docutils literal notranslate"><span class="pre">datafusion_query_planner</span></code> PyCapsule wrapping the session’s
+current planner, exported for a foreign planner to delegate to.</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">fallback</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">__datafusion_query_planner__</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="nb">type</span><span class="p">(</span><span class="n">fallback</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span>
+<span class="go">'PyCapsule'</span>
+</pre></div>
+</div>
+<p>Both spellings work, since the argument is ignored:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="nb">type</span><span class="p">(</span><span class="n">ctx</span><span class="o">.</span><span class="n">__datafusion_query_planner__</span><span class="p">(</span><span class="n">ctx</span><span class="p">))</span><span class="o">.</span><span class="vm">__name__</span>
+<span class="go">'PyCapsule'</span>
+</pre></div>
+</div>
</dd></dl>
<dl class="py method">
@@ -1083,9 +1235,32 @@ a query against a CSV data source using the <code class="xref py py-class docuti
<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">_convert_table_partition_cols</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">table_partition_cols</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">tuple</span [...]
<dd></dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext._register_object_store_for_path">
+<span class="sig-name descname"><span class="pre">_register_object_store_for_path</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><spa [...]
+<dd><p>Parse a URL path and register the given object store for its scheme and host.</p>
+<p>This is a convenience helper used by methods like
+<a class="reference internal" href="#datafusion.context.SessionContext.register_parquet" title="datafusion.context.SessionContext.register_parquet"><code class="xref py py-meth docutils literal notranslate"><span class="pre">register_parquet()</span></code></a> and <a class="reference internal" href="#datafusion.context.SessionContext.read_parquet" title="datafusion.context.SessionContext.read_parquet"><code class="xref py py-meth docutils literal notranslate"><span class="pre">read_parq [...]
+automatically register an object store when an <code class="docutils literal notranslate"><span class="pre">object_store</span></code>
+parameter is provided.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>path</strong> – A URL-style path (e.g. <code class="docutils literal notranslate"><span class="pre">"s3://bucket/key.parquet"</span></code> or
+<code class="docutils literal notranslate"><span class="pre">"file:///tmp/data.parquet"</span></code>).</p></li>
+<li><p><strong>store</strong> – An object store instance to register.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Raises<span class="colon">:</span></dt>
+<dd class="field-even"><p><strong>ValueError</strong> – If the path does not contain a URL scheme, or if
+a non-file scheme is missing a host/bucket component.</p>
+</dd>
+</dl>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.add_physical_optimizer_rule">
-<span class="sig-name descname"><span class="pre">add_physical_optimizer_rule</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">rule</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#datafusion.context.PhysicalOptimizerRuleExportable" title="datafusion.context.PhysicalOptimizerRuleExportable"><span class="pre">PhysicalOptimizerRuleExportable</span></ [...]
+<span class="sig-name descname"><span class="pre">add_physical_optimizer_rule</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">rule</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../extensions/index.html#datafusion.extensions.PhysicalOptimizerRuleExportable" title="datafusion.extensions.PhysicalOptimizerRuleExportable"><span class="pre">datafusion [...]
<dd><p>Append a user-defined physical optimizer rule to the session.</p>
<p>The rule is imported via its <code class="docutils literal notranslate"><span class="pre">__datafusion_physical_optimizer_rule__</span></code>
PyCapsule, typically produced by a separate compiled extension. The
@@ -1095,7 +1270,8 @@ and catalogs are preserved.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>rule</strong> – Object exposing <code class="docutils literal notranslate"><span class="pre">__datafusion_physical_optimizer_rule__</span></code>,
-a <a class="reference internal" href="#datafusion.context.PhysicalOptimizerRuleExportable" title="datafusion.context.PhysicalOptimizerRuleExportable"><code class="xref py py-class docutils literal notranslate"><span class="pre">PhysicalOptimizerRuleExportable</span></code></a>.</p>
+a
+<a class="reference internal" href="../extensions/index.html#datafusion.extensions.PhysicalOptimizerRuleExportable" title="datafusion.extensions.PhysicalOptimizerRuleExportable"><code class="xref py py-class docutils literal notranslate"><span class="pre">PhysicalOptimizerRuleExportable</span></code></a>.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
@@ -1285,8 +1461,45 @@ needed for SQL queries.</p>
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.execute">
-<span class="sig-name descname"><span class="pre">execute</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">plan</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../plan/index.html#datafusion.plan.ExecutionPlan" title="datafusion.plan.ExecutionPlan"><span class="pre">datafusion.plan.ExecutionPlan</span></a></span></em>, <em class="sig-param"><span cl [...]
-<dd><p>Execute the <code class="docutils literal notranslate"><span class="pre">plan</span></code> and return the results.</p>
+<span class="sig-name descname"><span class="pre">execute</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">plan</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../plan/index.html#datafusion.plan.ExecutionPlan" title="datafusion.plan.ExecutionPlan"><span class="pre">datafusion.plan.ExecutionPlan</span></a></span></em>, <em class="sig-param"><span cl [...]
+<dd><p>Execute a single partition of <code class="docutils literal notranslate"><span class="pre">plan</span></code> and stream its batches.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>plan</strong> – The physical plan to execute.</p></li>
+<li><p><strong>partition</strong> – Index of the partition to execute, in
+<code class="docutils literal notranslate"><span class="pre">range(plan.partition_count)</span></code>.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>A stream over the record batches that partition produces.</p>
+</dd>
+<dt class="field-odd">Raises<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>ValueError</strong> – If <code class="docutils literal notranslate"><span class="pre">partition</span></code> is not a valid index for <code class="docutils literal notranslate"><span class="pre">plan</span></code>.</p></li>
+<li><p><strong>OverflowError</strong> – If <code class="docutils literal notranslate"><span class="pre">partition</span></code> is negative, or too large to fit a
+platform-sized unsigned integer.</p></li>
+</ul>
+</dd>
+</dl>
+<p>Example usage:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
+<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">register_record_batches</span><span class="p">(</span>
+<span class="gp">... </span> <span class="s2">"t"</span><span class="p">,</span> <span class="p">[[</span><span class="n">pa</span><span class="o">.</span><span class="n">record_batch</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]})],</span> <span class="p">[</span><span class="n">pa</span><span class="o">.</span>< [...]
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">plan</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"select a from t"</span><span class="p">)</span><span class="o">.</span><span class="n">execution_plan</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">plan</span><span class="o">.</span><span class="n">partition_count</span>
+<span class="go">2</span>
+<span class="gp">>>> </span><span class="nb">sum</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">batch</span><span class="o">.</span><span class="n">to_pyarrow</span><span class="p">()</span><span class="o">.</span><span class="n">num_rows</span>
+<span class="gp">... </span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">plan</span><span class="o">.</span><span class="n">partition_count</span><span class="p">)</span>
+<span class="gp">... </span> <span class="k">for</span> <span class="n">batch</span> <span class="ow">in</span> <span class="n">ctx</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">plan</span><span class="p">,</span> <span class="n">p</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="go">3</span>
+</pre></div>
+</div>
</dd></dl>
<dl class="py method">
@@ -1417,6 +1630,37 @@ a struct array.</p>
</dl>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.logical_extension_codec_ids">
+<span class="sig-name descname"><span class="pre">logical_extension_codec_ids</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.context.SessionContext [...]
+<dd><p>List the logical extension codecs installed on this session.</p>
+<p>Returns the identity of each installed codec, in install order. Those
+identities are what encoding stamps onto a payload and what decoding
+dispatches on, so this is how to check which library owns a plan and
+whether a session is able to decode one.</p>
+<p>DataFusion’s own default codec is not listed. It handles whatever no
+installed codec claims, and it carries no identity to list.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Returns<span class="colon">:</span></dt>
+<dd class="field-odd"><p>The installed codec ids, in install order. Empty if none are
+installed.</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span>
+<span class="gp">>>> </span><span class="n">host</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">host</span><span class="o">.</span><span class="n">logical_extension_codec_ids</span><span class="p">()</span>
+<span class="go">[]</span>
+<span class="gp">>>> </span><span class="n">capsule</span> <span class="o">=</span> <span class="n">host</span><span class="o">.</span><span class="n">__datafusion_logical_extension_codec__</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span><span class="o">.</span><span class="n">with_logical_extension_codec</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">capsule</span><span class="p">,</span> <span class="n">codec_id</span><span class="o">=</span><span class="s2">"my_library.Codec"</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">logical_extension_codec_ids</span><span class="p">()</span>
+<span class="go">['my_library.Codec']</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.parse_capacity_limit">
<em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">parse_capacity_limit</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">config_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">limit</span></span><span cl [...]
@@ -1464,9 +1708,35 @@ limit belongs to. Use this helper when constructing a
</div>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.physical_extension_codec_ids">
+<span class="sig-name descname"><span class="pre">physical_extension_codec_ids</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#datafusion.context.SessionContex [...]
+<dd><p>List the physical extension codecs installed on this session.</p>
+<p>See <a class="reference internal" href="#datafusion.context.SessionContext.logical_extension_codec_ids" title="datafusion.context.SessionContext.logical_extension_codec_ids"><code class="xref py py-meth docutils literal notranslate"><span class="pre">logical_extension_codec_ids()</span></code></a>.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Returns<span class="colon">:</span></dt>
+<dd class="field-odd"><p>The installed codec ids, in install order. Empty if none are
+installed.</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span>
+<span class="gp">>>> </span><span class="n">host</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">host</span><span class="o">.</span><span class="n">physical_extension_codec_ids</span><span class="p">()</span>
+<span class="go">[]</span>
+<span class="gp">>>> </span><span class="n">capsule</span> <span class="o">=</span> <span class="n">host</span><span class="o">.</span><span class="n">__datafusion_physical_extension_codec__</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span><span class="o">.</span><span class="n">with_physical_extension_codec</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">capsule</span><span class="p">,</span> <span class="n">codec_id</span><span class="o">=</span><span class="s2">"my_library.PhysicalCodec"</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">physical_extension_codec_ids</span><span class="p">()</span>
+<span class="go">['my_library.PhysicalCodec']</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.read_arrow">
-<span class="sig-name descname"><span class="pre">read_arrow</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span cla [...]
+<span class="sig-name descname"><span class="pre">read_arrow</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span cla [...]
<dd><p>Create a <code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code> for reading an Arrow IPC data source.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
@@ -1475,6 +1745,9 @@ limit belongs to. Use this helper when constructing a
<li><p><strong>schema</strong> – The data source schema.</p></li>
<li><p><strong>file_extension</strong> – File extension to select.</p></li>
<li><p><strong>file_partition_cols</strong> – Partition columns.</p></li>
+<li><p><strong>object_store</strong> – A pre-configured object store instance to use for
+accessing the file. When provided, the store is automatically
+registered for the URL scheme and host parsed from <code class="docutils literal notranslate"><span class="pre">path</span></code>.</p></li>
</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
@@ -1533,7 +1806,7 @@ limit belongs to. Use this helper when constructing a
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.read_avro">
-<span class="sig-name descname"><span class="pre">read_avro</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span clas [...]
+<span class="sig-name descname"><span class="pre">read_avro</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span clas [...]
<dd><p>Create a <code class="xref py py-class docutils literal notranslate"><span class="pre">DataFrame</span></code> for reading Avro data source.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
@@ -1542,6 +1815,9 @@ limit belongs to. Use this helper when constructing a
<li><p><strong>schema</strong> – The data source schema.</p></li>
<li><p><strong>file_partition_cols</strong> – Partition columns.</p></li>
<li><p><strong>file_extension</strong> – File extension to select.</p></li>
+<li><p><strong>object_store</strong> – A pre-configured object store instance to use for
+accessing the file. When provided, the store is automatically
+registered for the URL scheme and host parsed from <code class="docutils literal notranslate"><span class="pre">path</span></code>.</p></li>
</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
@@ -1604,7 +1880,7 @@ an anonymous <code class="xref py py-class docutils literal notranslate"><span c
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.read_csv">
-<span class="sig-name descname"><span class="pre">read_csv</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span><span class="w"> </span><span class="p"><span class="pre">|</sp [...]
+<span class="sig-name descname"><span class="pre">read_csv</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span><span class="w"> </span><span class="p"><span class="pre">|</sp [...]
<dd><p>Read a CSV data source.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
@@ -1624,6 +1900,9 @@ selected for data input.</p></li>
<li><p><strong>file_compression_type</strong> – File compression type.</p></li>
<li><p><strong>options</strong> – Set advanced options for CSV reading. This cannot be
combined with any of the other options in this method.</p></li>
+<li><p><strong>object_store</strong> – A pre-configured object store instance to use for
+accessing the file. When provided, the store is automatically
+registered for the URL scheme and host parsed from <code class="docutils literal notranslate"><span class="pre">path</span></code>.</p></li>
</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
@@ -1644,7 +1923,7 @@ combined with any of the other options in this method.</p></li>
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.read_json">
-<span class="sig-name descname"><span class="pre">read_json</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span clas [...]
+<span class="sig-name descname"><span class="pre">read_json</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span clas [...]
<dd><p>Read a line-delimited JSON data source.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
@@ -1657,6 +1936,9 @@ files for schema inference if needed.</p></li>
selected for data input.</p></li>
<li><p><strong>table_partition_cols</strong> – Partition columns.</p></li>
<li><p><strong>file_compression_type</strong> – File compression type.</p></li>
+<li><p><strong>object_store</strong> – A pre-configured object store instance to use for
+accessing the file. When provided, the store is automatically
+registered for the URL scheme and host parsed from <code class="docutils literal notranslate"><span class="pre">path</span></code>.</p></li>
</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
@@ -1667,7 +1949,7 @@ selected for data input.</p></li>
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.read_parquet">
-<span class="sig-name descname"><span class="pre">read_parquet</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span c [...]
+<span class="sig-name descname"><span class="pre">read_parquet</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">pathlib.Path</span></span></em>, <em class="sig-param"><span class="n"><span c [...]
<dd><p>Read a Parquet source into a <code class="xref py py-class docutils literal notranslate"><span class="pre">Dataframe</span></code>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
@@ -1687,12 +1969,43 @@ file.</p></li>
<li><p><strong>file_sort_order</strong> – Sort order for the file. Each sort key can be
specified as a column name (<code class="docutils literal notranslate"><span class="pre">str</span></code>), an expression
(<code class="docutils literal notranslate"><span class="pre">Expr</span></code>), or a <code class="docutils literal notranslate"><span class="pre">SortExpr</span></code>.</p></li>
+<li><p><strong>object_store</strong> – A pre-configured object store instance (e.g.
+<a class="reference internal" href="../object_store/index.html#datafusion.object_store.AmazonS3" title="datafusion.object_store.AmazonS3"><code class="xref py py-class docutils literal notranslate"><span class="pre">AmazonS3</span></code></a>,
+<a class="reference internal" href="../object_store/index.html#datafusion.object_store.GoogleCloud" title="datafusion.object_store.GoogleCloud"><code class="xref py py-class docutils literal notranslate"><span class="pre">GoogleCloud</span></code></a>,
+<a class="reference internal" href="../object_store/index.html#datafusion.object_store.MicrosoftAzure" title="datafusion.object_store.MicrosoftAzure"><code class="xref py py-class docutils literal notranslate"><span class="pre">MicrosoftAzure</span></code></a>) to use
+for accessing the file. When provided, the store is
+automatically registered for the URL scheme and host parsed
+from <code class="docutils literal notranslate"><span class="pre">path</span></code>, removing the need to call
+<a class="reference internal" href="#datafusion.context.SessionContext.register_object_store" title="datafusion.context.SessionContext.register_object_store"><code class="xref py py-meth docutils literal notranslate"><span class="pre">register_object_store()</span></code></a> separately. This is
+especially useful in multi-threaded environments where
+setting credentials via <code class="docutils literal notranslate"><span class="pre">os.environ</span></code> is not thread-safe.</p></li>
</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>DataFrame representation of the read Parquet files</p>
</dd>
</dl>
+<p class="rubric">Examples</p>
+<p>Read a local Parquet file:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">datafusion</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">datafusion</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">read_parquet</span><span class="p">(</span><span class="s2">"data.parquet"</span><span class="p">)</span>
+</pre></div>
+</div>
+<p>Read from S3 with inline credentials (thread-safe):</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion.object_store</span><span class="w"> </span><span class="kn">import</span> <span class="n">AmazonS3</span>
+<span class="gp">>>> </span><span class="n">store</span> <span class="o">=</span> <span class="n">AmazonS3</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">bucket_name</span><span class="o">=</span><span class="s2">"my-bucket"</span><span class="p">,</span>
+<span class="gp">... </span> <span class="n">region</span><span class="o">=</span><span class="s2">"us-east-1"</span><span class="p">,</span>
+<span class="gp">... </span> <span class="n">access_key_id</span><span class="o">=</span><span class="s2">"..."</span><span class="p">,</span>
+<span class="gp">... </span> <span class="n">secret_access_key</span><span class="o">=</span><span class="s2">"..."</span><span class="p">,</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">df</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">read_parquet</span><span class="p">(</span>
+<span class="gp">... </span> <span class="s2">"s3://my-bucket/data.parquet"</span><span class="p">,</span>
+<span class="gp">... </span> <span class="n">object_store</span><span class="o">=</span><span class="n">store</span><span class="p">,</span>
+<span class="gp">... </span><span class="p">)</span>
+</pre></div>
+</div>
</dd></dl>
<dl class="py method">
@@ -1714,7 +2027,7 @@ specified as a column name (<code class="docutils literal notranslate"><span cla
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.register_arrow">
-<span class="sig-name descname"><span class="pre">register_arrow</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="p [...]
+<span class="sig-name descname"><span class="pre">register_arrow</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="p [...]
<dd><p>Register an Arrow IPC file as a table.</p>
<p>The registered table can be referenced from SQL statements executed
against this context.</p>
@@ -1726,6 +2039,9 @@ against this context.</p>
<li><p><strong>schema</strong> – The data source schema.</p></li>
<li><p><strong>file_extension</strong> – File extension to select.</p></li>
<li><p><strong>table_partition_cols</strong> – Partition columns.</p></li>
+<li><p><strong>object_store</strong> – A pre-configured object store instance to use for
+accessing the file. When provided, the store is automatically
+registered for the URL scheme and host parsed from <code class="docutils literal notranslate"><span class="pre">path</span></code>.</p></li>
</ul>
</dd>
</dl>
@@ -1787,7 +2103,7 @@ against this context.</p>
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.register_avro">
-<span class="sig-name descname"><span class="pre">register_avro</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pr [...]
+<span class="sig-name descname"><span class="pre">register_avro</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pr [...]
<dd><p>Register an Avro file as a table.</p>
<p>The registered table can be referenced from SQL statement executed against
this context.</p>
@@ -1799,6 +2115,9 @@ this context.</p>
<li><p><strong>schema</strong> – The data source schema.</p></li>
<li><p><strong>file_extension</strong> – File extension to select.</p></li>
<li><p><strong>table_partition_cols</strong> – Partition columns.</p></li>
+<li><p><strong>object_store</strong> – A pre-configured object store instance to use for
+accessing the file. When provided, the store is automatically
+registered for the URL scheme and host parsed from <code class="docutils literal notranslate"><span class="pre">path</span></code>.</p></li>
</ul>
</dd>
</dl>
@@ -1845,7 +2164,7 @@ this context.</p>
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.register_csv">
-<span class="sig-name descname"><span class="pre">register_csv</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre [...]
+<span class="sig-name descname"><span class="pre">register_csv</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre [...]
<dd><p>Register a CSV file as a table.</p>
<p>The registered table can be referenced from SQL statement executed against.</p>
<dl class="field-list simple">
@@ -1866,6 +2185,9 @@ selected for data input.</p></li>
<li><p><strong>file_compression_type</strong> – File compression type.</p></li>
<li><p><strong>options</strong> – Set advanced options for CSV reading. This cannot be
combined with any of the other options in this method.</p></li>
+<li><p><strong>object_store</strong> – A pre-configured object store instance to use for
+accessing the file. When provided, the store is automatically
+registered for the URL scheme and host parsed from <code class="docutils literal notranslate"><span class="pre">path</span></code>.</p></li>
</ul>
</dd>
</dl>
@@ -1887,7 +2209,7 @@ combined with any of the other options in this method.</p></li>
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.register_json">
-<span class="sig-name descname"><span class="pre">register_json</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pr [...]
+<span class="sig-name descname"><span class="pre">register_json</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pr [...]
<dd><p>Register a JSON file as a table.</p>
<p>The registered table can be referenced from SQL statement executed
against this context.</p>
@@ -1903,6 +2225,9 @@ files for schema inference if needed.</p></li>
selected for data input.</p></li>
<li><p><strong>table_partition_cols</strong> – Partition columns.</p></li>
<li><p><strong>file_compression_type</strong> – File compression type.</p></li>
+<li><p><strong>object_store</strong> – A pre-configured object store instance to use for
+accessing the file. When provided, the store is automatically
+registered for the URL scheme and host parsed from <code class="docutils literal notranslate"><span class="pre">path</span></code>.</p></li>
</ul>
</dd>
</dl>
@@ -1948,7 +2273,7 @@ specified as a column name (<code class="docutils literal notranslate"><span cla
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.register_parquet">
-<span class="sig-name descname"><span class="pre">register_parquet</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class= [...]
+<span class="sig-name descname"><span class="pre">register_parquet</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class= [...]
<dd><p>Register a Parquet file as a table.</p>
<p>The registered table can be referenced from SQL statement executed
against this context.</p>
@@ -1969,9 +2294,41 @@ conflicts due to metadata.</p></li>
<li><p><strong>file_sort_order</strong> – Sort order for the file. Each sort key can be
specified as a column name (<code class="docutils literal notranslate"><span class="pre">str</span></code>), an expression
(<code class="docutils literal notranslate"><span class="pre">Expr</span></code>), or a <code class="docutils literal notranslate"><span class="pre">SortExpr</span></code>.</p></li>
+<li><p><strong>object_store</strong> – A pre-configured object store instance (e.g.
+<a class="reference internal" href="../object_store/index.html#datafusion.object_store.AmazonS3" title="datafusion.object_store.AmazonS3"><code class="xref py py-class docutils literal notranslate"><span class="pre">AmazonS3</span></code></a>,
+<a class="reference internal" href="../object_store/index.html#datafusion.object_store.GoogleCloud" title="datafusion.object_store.GoogleCloud"><code class="xref py py-class docutils literal notranslate"><span class="pre">GoogleCloud</span></code></a>,
+<a class="reference internal" href="../object_store/index.html#datafusion.object_store.MicrosoftAzure" title="datafusion.object_store.MicrosoftAzure"><code class="xref py py-class docutils literal notranslate"><span class="pre">MicrosoftAzure</span></code></a>) to use
+for accessing the file. When provided, the store is
+automatically registered for the URL scheme and host parsed
+from <code class="docutils literal notranslate"><span class="pre">path</span></code>, removing the need to call
+<a class="reference internal" href="#datafusion.context.SessionContext.register_object_store" title="datafusion.context.SessionContext.register_object_store"><code class="xref py py-meth docutils literal notranslate"><span class="pre">register_object_store()</span></code></a> separately. This is
+especially useful in multi-threaded environments where
+setting credentials via <code class="docutils literal notranslate"><span class="pre">os.environ</span></code> is not thread-safe.</p></li>
</ul>
</dd>
</dl>
+<p class="rubric">Examples</p>
+<p>Register a local Parquet file:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">datafusion</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">datafusion</span><span class="o">.</span><span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">register_parquet</span><span class="p">(</span><span class="s2">"my_table"</span><span class="p">,</span> <span class="s2">"data.parquet"</span><span class="p">)</span>
+</pre></div>
+</div>
+<p>Register from S3 with inline credentials (thread-safe):</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion.object_store</span><span class="w"> </span><span class="kn">import</span> <span class="n">AmazonS3</span>
+<span class="gp">>>> </span><span class="n">store</span> <span class="o">=</span> <span class="n">AmazonS3</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">bucket_name</span><span class="o">=</span><span class="s2">"my-bucket"</span><span class="p">,</span>
+<span class="gp">... </span> <span class="n">region</span><span class="o">=</span><span class="s2">"us-east-1"</span><span class="p">,</span>
+<span class="gp">... </span> <span class="n">access_key_id</span><span class="o">=</span><span class="s2">"..."</span><span class="p">,</span>
+<span class="gp">... </span> <span class="n">secret_access_key</span><span class="o">=</span><span class="s2">"..."</span><span class="p">,</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">register_parquet</span><span class="p">(</span>
+<span class="gp">... </span> <span class="s2">"my_table"</span><span class="p">,</span>
+<span class="gp">... </span> <span class="s2">"s3://my-bucket/data.parquet"</span><span class="p">,</span>
+<span class="gp">... </span> <span class="n">object_store</span><span class="o">=</span><span class="n">store</span><span class="p">,</span>
+<span class="gp">... </span><span class="p">)</span>
+</pre></div>
+</div>
</dd></dl>
<dl class="py method">
@@ -2008,7 +2365,7 @@ this context.</p>
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.register_table_factory">
-<span class="sig-name descname"><span class="pre">register_table_factory</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">format</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">factory</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n">< [...]
+<span class="sig-name descname"><span class="pre">register_table_factory</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">format</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">factory</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n">< [...]
<dd><p>Register a <a class="reference internal" href="../index.html#datafusion.TableProviderFactoryExportable" title="datafusion.TableProviderFactoryExportable"><code class="xref py py-class docutils literal notranslate"><span class="pre">TableProviderFactoryExportable</span></code></a>.</p>
<p>The registered factory can be referenced from SQL DDL statements executed
against this context.</p>
@@ -2105,6 +2462,56 @@ against this context.</p>
</div>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.set_query_planner">
+<span class="sig-name descname"><span class="pre">set_query_planner</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">planner</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../extensions/index.html#datafusion.extensions.QueryPlannerExportable" title="datafusion.extensions.QueryPlannerExportable"><span class="pre">datafusion.extensions.QueryPlannerE [...]
+<dd><p>Install a custom query planner on this session.</p>
+<p>The planner is imported through its <code class="docutils literal notranslate"><span class="pre">__datafusion_query_planner__</span></code>
+PyCapsule, in the same way
+<a class="reference internal" href="#datafusion.context.SessionContext.add_physical_optimizer_rule" title="datafusion.context.SessionContext.add_physical_optimizer_rule"><code class="xref py py-meth docutils literal notranslate"><span class="pre">add_physical_optimizer_rule()</span></code></a> installs a rule.</p>
+<p>Returns nothing, because the planner lives in the session rather than
+in a handle on it — installing one is visible to every context sharing
+that session, including ones an earlier <code class="docutils literal notranslate"><span class="pre">with_*</span></code> call returned. See
+<a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionContext</span></code></a>.</p>
+<p>A session holds exactly one planner, so calling this again <strong>replaces</strong>
+the previous one rather than layering. To chain planners, have the new
+planner wrap the capsule from
+<a class="reference internal" href="#datafusion.context.SessionContext.__datafusion_query_planner__" title="datafusion.context.SessionContext.__datafusion_query_planner__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__datafusion_query_planner__()</span></code></a>, captured <em>before</em>
+the new planner is installed.</p>
+<p>Install any extension codecs before a layered planner; the rebuild that
+follows a later codec install does not reach the fallback inside one.
+See <a class="reference internal" href="../../../extension-guide/query-planners.html#planner-codec-rebinding"><span class="std std-ref">Install codecs before a layered planner</span></a>, or prefer
+<a class="reference internal" href="#datafusion.context.SessionContext.with_extensions" title="datafusion.context.SessionContext.with_extensions"><code class="xref py py-meth docutils literal notranslate"><span class="pre">with_extensions()</span></code></a>, which cannot capture a partial chain.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>planner</strong> – Object exposing <code class="docutils literal notranslate"><span class="pre">__datafusion_query_planner__</span></code> (see
+<a class="reference internal" href="../extensions/index.html#datafusion.extensions.QueryPlannerExportable" title="datafusion.extensions.QueryPlannerExportable"><code class="xref py py-class docutils literal notranslate"><span class="pre">QueryPlannerExportable</span></code></a>) or a raw
+<code class="docutils literal notranslate"><span class="pre">datafusion_query_planner</span></code> PyCapsule.</p>
+</dd>
+<dt class="field-even">Raises<span class="colon">:</span></dt>
+<dd class="field-even"><p><strong>ValueError</strong> – If the capsule is not named <code class="docutils literal notranslate"><span class="pre">datafusion_query_planner</span></code>.</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<p>A session exports its own planner, which is what you capture to
+wrap:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">fallback</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">__datafusion_query_planner__</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="nb">type</span><span class="p">(</span><span class="n">fallback</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span>
+<span class="go">'PyCapsule'</span>
+</pre></div>
+</div>
+<p>Skipped here (needs a built extension library):</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">my_extension</span><span class="w"> </span><span class="kn">import</span> <span class="n">DistributedQueryPlanner</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">set_query_planner</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">DistributedQueryPlanner</span><span class="p">(</span><span class="n">fallback</span><span class="o">=</span><span class="n">fallback</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT * FROM remote_table"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.sql">
<span class="sig-name descname"><span class="pre">sql</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">query</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">options</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference i [...]
@@ -2335,31 +2742,196 @@ functions.</p>
</div>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.with_extensions">
+<span class="sig-name descname"><span class="pre">with_extensions</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">extensions</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../extensions/index.html#datafusion.extensions.SessionComponentsExportable" title="datafusion.extensions.SessionComponentsExpor [...]
+<dd><p>Create a new session context with the given extension bundles.</p>
+<p>This is the preferred way to install extension codecs and query
+planners, because it removes the ordering question that installing them
+by hand creates.</p>
+<p>Each argument is called twice, in two phases:</p>
+<ol class="arabic simple">
+<li><p><code class="docutils literal notranslate"><span class="pre">__datafusion_session_components__(ctx)</span></code> on every extension, then
+all the returned codecs are installed at once.</p></li>
+<li><p><code class="docutils literal notranslate"><span class="pre">__datafusion_session_planner__(ctx,</span> <span class="pre">fallback)</span></code> on every
+extension, <strong>in argument order</strong>, each handed the planner built so
+far and the context carrying every bundle’s codecs. The last
+extension listed ends up outermost.</p></li>
+</ol>
+<p>An extension implements either hook or both. Return <code class="docutils literal notranslate"><span class="pre">None</span></code> from the
+planner hook to contribute no planner; see
+<a class="reference internal" href="../extensions/index.html#datafusion.extensions.SessionPlannerExportable" title="datafusion.extensions.SessionPlannerExportable"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionPlannerExportable</span></code></a>.</p>
+<p>Nothing is written to the session until every hook has returned and
+every capsule has been validated, so a hook that raises leaves the
+session as it was. A hook that <em>mutates</em> the context it is handed —
+registering a table, say — is not rolled back, which is why bundle
+objects must be configuration-only. See
+<a class="reference internal" href="../../../extension-guide/bundles.html#extension-bundles-transaction"><span class="std std-ref">Failure and rollback</span></a>.</p>
+<p>Shares its session with this context — see <a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionContext</span></code></a>.</p>
+<p>See <a class="reference internal" href="../../../extension-guide/bundles.html#extension-bundles"><span class="std std-ref">Extension bundles</span></a> in the online documentation for why the
+phases are split, how to contribute a bundle’s two halves at different
+positions, and a worked Rust implementation.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>extensions</strong> – Extension bundles to install. Order is irrelevant for
+codecs and significant for planners, which nest in this order
+with the last one outermost. Passing none installs nothing and
+returns a handle on this session, so a caller assembling the
+list at runtime need not special-case it being empty.</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>A new context with all extension components installed.</p>
+</dd>
+<dt class="field-odd">Raises<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>TypeError</strong> – If an argument implements neither hook, if a hook
+returns the wrong type, or if a codec is contributed as a bare
+<code class="docutils literal notranslate"><span class="pre">PyCapsule</span></code> rather than an object exposing the getter.</p></li>
+<li><p><strong>ValueError</strong> – If two codecs claim the same id, or a getter returns a
+capsule of the wrong kind. See
+<a class="reference internal" href="#datafusion.context.SessionContext.with_logical_extension_codec" title="datafusion.context.SessionContext.with_logical_extension_codec"><code class="xref py py-meth docutils literal notranslate"><span class="pre">with_logical_extension_codec()</span></code></a> for how ids are
+assigned.</p></li>
+</ul>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<p>The returned handle is a different object sharing one session, and
+an empty call is legal:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">derived</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">with_extensions</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">derived</span> <span class="ow">is</span> <span class="n">ctx</span>
+<span class="go">False</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">from_pydict</span><span class="p">({</span><span class="s2">"a"</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]},</span> <span class="n">name</span><span class="o">=</span><span class="s2">"t"</span><span class="p">)</span>
+<span class="go">DataFrame()...</span>
+<span class="gp">>>> </span><span class="n">derived</span><span class="o">.</span><span class="n">table_exist</span><span class="p">(</span><span class="s2">"t"</span><span class="p">)</span>
+<span class="go">True</span>
+<span class="gp">>>> </span><span class="n">derived</span><span class="o">.</span><span class="n">logical_extension_codec_ids</span><span class="p">()</span>
+<span class="go">[]</span>
+</pre></div>
+</div>
+<p>A runnable multi-bundle example, showing what a bundle returns and
+how its codec ids accumulate, is in <a class="reference internal" href="../../../extension-guide/bundles.html#extension-bundles"><span class="std std-ref">Extension bundles</span></a>.</p>
+<p>Real usage. Skipped here (needs a built extension library); run
+verbatim by <code class="docutils literal notranslate"><span class="pre">test_with_extensions_docstring_example_still_runs</span></code>.</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">my_extension</span><span class="w"> </span><span class="kn">import</span> <span class="n">DistributedEngineExtension</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span><span class="o">.</span><span class="n">with_extensions</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">DistributedEngineExtension</span><span class="p">(</span><span class="s2">"scheduler:50050"</span><span class="p">)</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">batches</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s2">"SELECT 1 AS n"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">batches</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">column</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="n">to_pylist</span><span class="p">()</span>
+<span class="go">[1]</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.with_logical_extension_codec">
-<span class="sig-name descname"><span class="pre">with_logical_extension_codec</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">codec</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../user_defined/index.html#datafusion.user_defined.LogicalExtensionCodecExportable" title="datafusion.user_defined.LogicalExtensionCodecExportable"><span class="pre">da [...]
-<dd><p>Create a new session context with specified codec.</p>
+<span class="sig-name descname"><span class="pre">with_logical_extension_codec</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">codec</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../user_defined/index.html#datafusion.user_defined.LogicalExtensionCodecExportable" title="datafusion.user_defined.LogicalExtensionCodecExportable"><span class="pre">da [...]
+<dd><p>Create a new session context with an additional logical codec.</p>
<p>Only FFI codecs are supported. Pass any object implementing
<code class="docutils literal notranslate"><span class="pre">__datafusion_logical_extension_codec__</span></code> (see
<a class="reference internal" href="../user_defined/index.html#datafusion.user_defined.LogicalExtensionCodecExportable" title="datafusion.user_defined.LogicalExtensionCodecExportable"><code class="xref py py-class docutils literal notranslate"><span class="pre">LogicalExtensionCodecExportable</span></code></a>).</p>
+<p>Codecs compose: each call appends the codec rather than replacing
+codecs installed earlier, so one session can carry codecs from several
+independent libraries and the order they are installed in does not
+affect decoding.</p>
+<p>A serialized plan records which codec wrote each payload, as a short id
+taken from the codec’s class.</p>
+<p>Shares its session with this context — see <a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionContext</span></code></a>.</p>
+<p>See <a class="reference internal" href="../../../extension-guide/codecs.html#extension-codec-ids"><span class="std std-ref">Codec ids</span></a> in the online documentation for how ids
+are assigned and what an extension codec has to implement, and
+<a class="reference internal" href="#datafusion.context.SessionContext.with_extensions" title="datafusion.context.SessionContext.with_extensions"><code class="xref py py-meth docutils literal notranslate"><span class="pre">with_extensions()</span></code></a> for installing a library’s codecs and planner
+together.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>codec</strong> – Object implementing <code class="docutils literal notranslate"><span class="pre">__datafusion_logical_extension_codec__</span></code>
+(see
+<a class="reference internal" href="../user_defined/index.html#datafusion.user_defined.LogicalExtensionCodecExportable" title="datafusion.user_defined.LogicalExtensionCodecExportable"><code class="xref py py-class docutils literal notranslate"><span class="pre">LogicalExtensionCodecExportable</span></code></a>),
+or a raw <code class="docutils literal notranslate"><span class="pre">datafusion_logical_extension_codec</span></code> PyCapsule.</p></li>
+<li><p><strong>codec_id</strong> – Overrides the id the codec’s payloads are tagged with.
+Normally unnecessary. Pass it when installing two instances of
+one class, which otherwise claim the same id, and when
+installing a bare <code class="docutils literal notranslate"><span class="pre">PyCapsule</span></code>: a capsule has no class to take
+an id from, so it is given a random <code class="docutils literal notranslate"><span class="pre">anon:</span></code> id that differs on
+every install, and plans it encodes can never be decoded
+elsewhere.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>A new context carrying this codec in addition to any already
+installed.</p>
+</dd>
+<dt class="field-odd">Raises<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>ValueError</strong> – If the resolved id is already installed on this session.</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<p>A context exports its own codec, which stands in here for a real
+library’s:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span>
+<span class="gp">>>> </span><span class="n">host</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">capsule</span> <span class="o">=</span> <span class="n">host</span><span class="o">.</span><span class="n">__datafusion_logical_extension_codec__</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span><span class="o">.</span><span class="n">with_logical_extension_codec</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">capsule</span><span class="p">,</span> <span class="n">codec_id</span><span class="o">=</span><span class="s2">"my_library.Codec"</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">logical_extension_codec_ids</span><span class="p">()</span>
+<span class="go">['my_library.Codec']</span>
+</pre></div>
+</div>
+<p>Without <code class="docutils literal notranslate"><span class="pre">codec_id</span></code> a bare capsule gets an anonymous id, which is
+fine only if its plans never leave this session:</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span><span class="o">.</span><span class="n">with_logical_extension_codec</span><span class="p">(</span><span class="n">capsule</span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">logical_extension_codec_ids</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"anon:"</span><span class="p">)</span>
+<span class="go">True</span>
+</pre></div>
+</div>
</dd></dl>
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.with_physical_extension_codec">
-<span class="sig-name descname"><span class="pre">with_physical_extension_codec</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">codec</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../user_defined/index.html#datafusion.user_defined.PhysicalExtensionCodecExportable" title="datafusion.user_defined.PhysicalExtensionCodecExportable"><span class="pre" [...]
-<dd><p>Create a new session context with the specified physical codec.</p>
+<span class="sig-name descname"><span class="pre">with_physical_extension_codec</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">codec</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="../user_defined/index.html#datafusion.user_defined.PhysicalExtensionCodecExportable" title="datafusion.user_defined.PhysicalExtensionCodecExportable"><span class="pre" [...]
+<dd><p>Create a new session context with an additional physical codec.</p>
<p>Only FFI codecs are supported. Pass any object implementing
<code class="docutils literal notranslate"><span class="pre">__datafusion_physical_extension_codec__</span></code> (see
<a class="reference internal" href="../user_defined/index.html#datafusion.user_defined.PhysicalExtensionCodecExportable" title="datafusion.user_defined.PhysicalExtensionCodecExportable"><code class="xref py py-class docutils literal notranslate"><span class="pre">PhysicalExtensionCodecExportable</span></code></a>).</p>
+<p>Composes and assigns an id exactly as
+<a class="reference internal" href="#datafusion.context.SessionContext.with_logical_extension_codec" title="datafusion.context.SessionContext.with_logical_extension_codec"><code class="xref py py-meth docutils literal notranslate"><span class="pre">with_logical_extension_codec()</span></code></a> does, including when to pass
+<code class="docutils literal notranslate"><span class="pre">codec_id</span></code> and what the returned context shares. See that method.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>codec</strong> – As <a class="reference internal" href="#datafusion.context.SessionContext.with_logical_extension_codec" title="datafusion.context.SessionContext.with_logical_extension_codec"><code class="xref py py-meth docutils literal notranslate"><span class="pre">with_logical_extension_codec()</span></code></a>, for the physical
+getter.</p></li>
+<li><p><strong>codec_id</strong> – As <a class="reference internal" href="#datafusion.context.SessionContext.with_logical_extension_codec" title="datafusion.context.SessionContext.with_logical_extension_codec"><code class="xref py py-meth docutils literal notranslate"><span class="pre">with_logical_extension_codec()</span></code></a>.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>A new context carrying this codec in addition to any already
+installed.</p>
+</dd>
+<dt class="field-odd">Raises<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>ValueError</strong> – If the resolved id is already installed on this session.</p>
+</dd>
+</dl>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span>
+<span class="gp">>>> </span><span class="n">host</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">capsule</span> <span class="o">=</span> <span class="n">host</span><span class="o">.</span><span class="n">__datafusion_physical_extension_codec__</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span><span class="o">.</span><span class="n">with_physical_extension_codec</span><span class="p">(</span>
+<span class="gp">... </span> <span class="n">capsule</span><span class="p">,</span> <span class="n">codec_id</span><span class="o">=</span><span class="s2">"my_library.PhysicalCodec"</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">physical_extension_codec_ids</span><span class="p">()</span>
+<span class="go">['my_library.PhysicalCodec']</span>
+</pre></div>
+</div>
</dd></dl>
<dl class="py method">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.with_python_udf_inlining">
<span class="sig-name descname"><span class="pre">with_python_udf_inlining</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">enabled</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon" [...]
<dd><p>Control whether Python UDFs are embedded in serialized expressions.</p>
-<p><code class="docutils literal notranslate"><span class="pre">enabled</span></code> is keyword-only and required: callers must pick a
-mode explicitly. Fresh sessions inline UDFs (<code class="docutils literal notranslate"><span class="pre">enabled=True</span></code>
-behavior) until this method overrides the toggle.</p>
<p>With <code class="docutils literal notranslate"><span class="pre">enabled=True</span></code>, serialized expressions carry the Python
code for any scalar, aggregate, or window UDFs they reference.
The receiver rebuilds the UDFs from those bytes and does not
@@ -2388,8 +2960,20 @@ This setting narrows only <code class="xref py py-meth docutils literal notransl
<code class="xref py py-func docutils literal notranslate"><span class="pre">pickle.loads()</span></code> on untrusted bytes remains unsafe
regardless of the toggle.</p>
</div>
-<p>Returns a new <a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionContext</span></code></a> with the toggle applied;
-the original session is unchanged.</p>
+<p>Shares its session with this context — see
+<a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionContext</span></code></a>. The original context’s own codec
+settings are unchanged.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>enabled</strong> – Whether to embed Python UDFs in serialized
+expressions. Keyword-only and required, so callers must
+pick a mode explicitly. Fresh sessions behave as
+<code class="docutils literal notranslate"><span class="pre">enabled=True</span></code> until this method overrides the toggle.</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>A new <a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionContext</span></code></a> with the toggle applied.</p>
+</dd>
+</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pyarrow</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pa</span>
<span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span><span class="p">,</span> <span class="n">Expr</span><span class="p">,</span> <span class="n">col</span><span class="p">,</span> <span class="n">udf</span>
@@ -2408,6 +2992,29 @@ the original session is unchanged.</p>
</div>
</dd></dl>
+<dl class="py property">
+<dt class="sig sig-object py" id="datafusion.context.SessionContext.__datafusion_codec_id__">
+<em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">__datafusion_codec_id__</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#datafusion.context.SessionContext.__datafusion_codec_id__" title="Link to this definition">#</a></dt>
+<dd><p>Identity this context carries when installed as an extension codec.</p>
+<p>A context can be installed on another session as an extension codec,
+which tags the payloads it writes with this string. It is unique per
+session, so two contexts can be installed on one session and a plan
+written through one will not be decoded by the other.</p>
+<p>Contexts derived from the same session report the same id, so only one
+of them can be installed on a given session. That is the intended
+answer: they are one session — see <a class="reference internal" href="#datafusion.context.SessionContext" title="datafusion.context.SessionContext"><code class="xref py py-class docutils literal notranslate"><span class="pre">SessionContext</span></code></a> — so
+their payloads would be indistinguishable on decode.</p>
+<p class="rubric">Examples</p>
+<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">from</span><span class="w"> </span><span class="nn">datafusion</span><span class="w"> </span><span class="kn">import</span> <span class="n">SessionContext</span>
+<span class="gp">>>> </span><span class="n">ctx</span> <span class="o">=</span> <span class="n">SessionContext</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">__datafusion_codec_id__</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"session:"</span><span class="p">)</span>
+<span class="go">True</span>
+<span class="gp">>>> </span><span class="n">ctx</span><span class="o">.</span><span class="n">__datafusion_codec_id__</span> <span class="o">==</span> <span class="n">SessionContext</span><span class="p">()</span><span class="o">.</span><span class="n">__datafusion_codec_id__</span>
+<span class="go">False</span>
+</pre></div>
+</div>
+</dd></dl>
+
<dl class="py attribute">
<dt class="sig sig-object py" id="datafusion.context.SessionContext.ctx">
<span class="sig-name descname"><span class="pre">ctx</span></span><a class="headerlink" href="#datafusion.context.SessionContext.ctx" title="Link to this definition">#</a></dt>
@@ -2420,7 +3027,19 @@ the original session is unchanged.</p>
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">datafusion.context.</span></span><span class="sig-name descname"><span class="pre">TableProviderExportable</span></span><a class="headerlink" href="#datafusion.context.TableProviderExportable" title="Link to this definition">#</a></dt>
<dd><p>Bases: <code class="xref py py-obj docutils literal notranslate"><span class="pre">Protocol</span></code></p>
<p>Type hint for object that has __datafusion_table_provider__ PyCapsule.</p>
-<p><a class="reference external" href="https://datafusion.apache.org/python/user-guide/io/table_provider.html">https://datafusion.apache.org/python/user-guide/io/table_provider.html</a></p>
+<p>See <a class="reference internal" href="../../../user-guide/io/table_provider.html#io-custom-table-provider"><span class="std std-ref">Custom Table Provider</span></a> for registering one, and
+<a class="reference internal" href="../../../extension-guide/table-providers.html#extension-providers"><span class="std std-ref">Providers and catalogs</span></a> for writing one.</p>
+<dl class="field-list simple">
... 22259 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org