You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ti...@apache.org on 2026/04/04 16:20:37 UTC
(datafusion-python) branch main updated: Add missing map functions (#1461)
This is an automated email from the ASF dual-hosted git repository.
timsaucer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-python.git
The following commit(s) were added to refs/heads/main by this push:
new 8a35caea Add missing map functions (#1461)
8a35caea is described below
commit 8a35caea9ed01492742738f161fa5b4459d69402
Author: Tim Saucer <ti...@gmail.com>
AuthorDate: Sat Apr 4 12:20:31 2026 -0400
Add missing map functions (#1461)
* Add map functions (make_map, map_keys, map_values, map_extract, map_entries, element_at)
Closes #1448
Co-Authored-By: Claude Opus 4.6 (1M context) <no...@anthropic.com>
* Add unit tests for map functions
Co-Authored-By: Claude Opus 4.6 (1M context) <no...@anthropic.com>
* Remove redundant pyo3 element_at function
element_at is already a Python-only alias for map_extract,
so the Rust binding is unnecessary.
Co-Authored-By: Claude Opus 4.6 (1M context) <no...@anthropic.com>
* Change make_map to accept a Python dictionary
make_map now takes a dict for the common case and also supports
separate keys/values lists for column expressions. Non-Expr keys
and values are automatically converted to literals.
Co-Authored-By: Claude Opus 4.6 (1M context) <no...@anthropic.com>
* Make map the primary function with make_map as alias
map() now supports three calling conventions matching upstream:
- map({"a": 1, "b": 2}) — from a Python dictionary
- map([keys], [values]) — two lists that get zipped
- map(k1, v1, k2, v2, ...) — variadic key-value pairs
Non-Expr keys and values are automatically converted to literals.
Co-Authored-By: Claude Opus 4.6 (1M context) <no...@anthropic.com>
* Improve map function docstrings
- Add examples for all three map() calling conventions
- Use clearer descriptions instead of jargon (no "zipped" or "variadic")
- Break map_keys/map_values/map_extract/map_entries examples into
two steps: create the map column first, then call the function
Co-Authored-By: Claude Opus 4.6 (1M context) <no...@anthropic.com>
* Remove map() in favor of make_map(), fix docstrings, add validation
- Remove map() function that shadowed Python builtin; make_map() is now
the sole entry point for creating map expressions
- Fix map_extract/element_at docstrings: missing keys return [None],
not an empty list (matches actual upstream behavior)
- Add length validation for the two-list calling convention
- Update all tests and docstring examples accordingly
Co-Authored-By: Claude Opus 4.6 (1M context) <no...@anthropic.com>
* Consolidate map function tests into parametrized groups
Reduce boilerplate by combining make_map construction tests and map
accessor function tests into two @pytest.mark.parametrize groups.
Co-Authored-By: Claude Opus 4.6 (1M context) <no...@anthropic.com>
* Docstring update
Co-authored-by: Nuno Faria <nu...@gmail.com>
* Docstring update
Co-authored-by: Nuno Faria <nu...@gmail.com>
* Simplify test for readability
Co-authored-by: Nuno Faria <nu...@gmail.com>
* Simplify test for readability
Co-authored-by: Nuno Faria <nu...@gmail.com>
---------
Co-authored-by: Claude Opus 4.6 (1M context) <no...@anthropic.com>
Co-authored-by: Nuno Faria <nu...@gmail.com>
---
crates/core/src/functions.rs | 20 ++++++
python/datafusion/functions.py | 158 +++++++++++++++++++++++++++++++++++++++++
python/tests/test_functions.py | 100 ++++++++++++++++++++++++++
3 files changed, 278 insertions(+)
diff --git a/crates/core/src/functions.rs b/crates/core/src/functions.rs
index 3f07da95..5e61b71b 100644
--- a/crates/core/src/functions.rs
+++ b/crates/core/src/functions.rs
@@ -93,6 +93,13 @@ fn array_cat(exprs: Vec<PyExpr>) -> PyExpr {
array_concat(exprs)
}
+#[pyfunction]
+fn make_map(keys: Vec<PyExpr>, values: Vec<PyExpr>) -> PyExpr {
+ let keys = keys.into_iter().map(|x| x.into()).collect();
+ let values = values.into_iter().map(|x| x.into()).collect();
+ datafusion::functions_nested::map::map(keys, values).into()
+}
+
#[pyfunction]
#[pyo3(signature = (array, element, index=None))]
fn array_position(array: PyExpr, element: PyExpr, index: Option<i64>) -> PyExpr {
@@ -678,6 +685,12 @@ array_fn!(cardinality, array);
array_fn!(flatten, array);
array_fn!(range, start stop step);
+// Map Functions
+array_fn!(map_keys, map);
+array_fn!(map_values, map);
+array_fn!(map_extract, map key);
+array_fn!(map_entries, map);
+
aggregate_function!(array_agg);
aggregate_function!(max);
aggregate_function!(min);
@@ -1142,6 +1155,13 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_wrapped(wrap_pyfunction!(flatten))?;
m.add_wrapped(wrap_pyfunction!(cardinality))?;
+ // Map Functions
+ m.add_wrapped(wrap_pyfunction!(make_map))?;
+ m.add_wrapped(wrap_pyfunction!(map_keys))?;
+ m.add_wrapped(wrap_pyfunction!(map_values))?;
+ m.add_wrapped(wrap_pyfunction!(map_extract))?;
+ m.add_wrapped(wrap_pyfunction!(map_entries))?;
+
// Window Functions
m.add_wrapped(wrap_pyfunction!(lead))?;
m.add_wrapped(wrap_pyfunction!(lag))?;
diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py
index f1ea3d25..3febb44e 100644
--- a/python/datafusion/functions.py
+++ b/python/datafusion/functions.py
@@ -140,6 +140,7 @@ __all__ = [
"degrees",
"dense_rank",
"digest",
+ "element_at",
"empty",
"encode",
"ends_with",
@@ -206,7 +207,12 @@ __all__ = [
"make_array",
"make_date",
"make_list",
+ "make_map",
"make_time",
+ "map_entries",
+ "map_extract",
+ "map_keys",
+ "map_values",
"max",
"md5",
"mean",
@@ -3458,6 +3464,158 @@ def empty(array: Expr) -> Expr:
return array_empty(array)
+# map functions
+
+
+def make_map(*args: Any) -> Expr:
+ """Returns a map expression.
+
+ Supports three calling conventions:
+
+ - ``make_map({"a": 1, "b": 2})`` — from a Python dictionary.
+ - ``make_map([keys], [values])`` — from a list of keys and a list of
+ their associated values. Both lists must be the same length.
+ - ``make_map(k1, v1, k2, v2, ...)`` — from alternating keys and their
+ associated values.
+
+ Keys and values that are not already :py:class:`~datafusion.expr.Expr`
+ are automatically converted to literal expressions.
+
+ Examples:
+ From a dictionary:
+
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> result = df.select(
+ ... dfn.functions.make_map({"a": 1, "b": 2}).alias("m"))
+ >>> result.collect_column("m")[0].as_py()
+ [('a', 1), ('b', 2)]
+
+ From two lists:
+
+ >>> df = ctx.from_pydict({"key": ["x", "y"], "val": [10, 20]})
+ >>> df = df.select(
+ ... dfn.functions.make_map(
+ ... [dfn.col("key")], [dfn.col("val")]
+ ... ).alias("m"))
+ >>> df.collect_column("m")[0].as_py()
+ [('x', 10)]
+
+ From alternating keys and values:
+
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> result = df.select(
+ ... dfn.functions.make_map("x", 1, "y", 2).alias("m"))
+ >>> result.collect_column("m")[0].as_py()
+ [('x', 1), ('y', 2)]
+ """
+ if len(args) == 1 and isinstance(args[0], dict):
+ key_list = list(args[0].keys())
+ value_list = list(args[0].values())
+ elif (
+ len(args) == 2 # noqa: PLR2004
+ and isinstance(args[0], list)
+ and isinstance(args[1], list)
+ ):
+ if len(args[0]) != len(args[1]):
+ msg = "make_map requires key and value lists to be the same length"
+ raise ValueError(msg)
+ key_list = args[0]
+ value_list = args[1]
+ elif len(args) >= 2 and len(args) % 2 == 0: # noqa: PLR2004
+ key_list = list(args[0::2])
+ value_list = list(args[1::2])
+ else:
+ msg = (
+ "make_map expects a dict, two lists, or an even number of "
+ "key-value arguments"
+ )
+ raise ValueError(msg)
+
+ key_exprs = [k if isinstance(k, Expr) else Expr.literal(k) for k in key_list]
+ val_exprs = [v if isinstance(v, Expr) else Expr.literal(v) for v in value_list]
+ return Expr(f.make_map([k.expr for k in key_exprs], [v.expr for v in val_exprs]))
+
+
+def map_keys(map: Expr) -> Expr:
+ """Returns a list of all keys in the map.
+
+ Examples:
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> df = df.select(
+ ... dfn.functions.make_map({"x": 1, "y": 2}).alias("m"))
+ >>> result = df.select(
+ ... dfn.functions.map_keys(dfn.col("m")).alias("keys"))
+ >>> result.collect_column("keys")[0].as_py()
+ ['x', 'y']
+ """
+ return Expr(f.map_keys(map.expr))
+
+
+def map_values(map: Expr) -> Expr:
+ """Returns a list of all values in the map.
+
+ Examples:
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> df = df.select(
+ ... dfn.functions.make_map({"x": 1, "y": 2}).alias("m"))
+ >>> result = df.select(
+ ... dfn.functions.map_values(dfn.col("m")).alias("vals"))
+ >>> result.collect_column("vals")[0].as_py()
+ [1, 2]
+ """
+ return Expr(f.map_values(map.expr))
+
+
+def map_extract(map: Expr, key: Expr) -> Expr:
+ """Returns the value for a given key in the map.
+
+ Returns ``[None]`` if the key is absent.
+
+ Examples:
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> df = df.select(
+ ... dfn.functions.make_map({"x": 1, "y": 2}).alias("m"))
+ >>> result = df.select(
+ ... dfn.functions.map_extract(
+ ... dfn.col("m"), dfn.lit("x")
+ ... ).alias("val"))
+ >>> result.collect_column("val")[0].as_py()
+ [1]
+ """
+ return Expr(f.map_extract(map.expr, key.expr))
+
+
+def map_entries(map: Expr) -> Expr:
+ """Returns a list of all entries (key-value struct pairs) in the map.
+
+ Examples:
+ >>> ctx = dfn.SessionContext()
+ >>> df = ctx.from_pydict({"a": [1]})
+ >>> df = df.select(
+ ... dfn.functions.make_map({"x": 1, "y": 2}).alias("m"))
+ >>> result = df.select(
+ ... dfn.functions.map_entries(dfn.col("m")).alias("entries"))
+ >>> result.collect_column("entries")[0].as_py()
+ [{'key': 'x', 'value': 1}, {'key': 'y', 'value': 2}]
+ """
+ return Expr(f.map_entries(map.expr))
+
+
+def element_at(map: Expr, key: Expr) -> Expr:
+ """Returns the value for a given key in the map.
+
+ Returns ``[None]`` if the key is absent.
+
+ See Also:
+ This is an alias for :py:func:`map_extract`.
+ """
+ return map_extract(map, key)
+
+
# aggregate functions
def approx_distinct(
expression: Expr,
diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py
index 74fcbffb..f25c6e78 100644
--- a/python/tests/test_functions.py
+++ b/python/tests/test_functions.py
@@ -668,6 +668,106 @@ def test_array_function_obj_tests(stmt, py_expr):
assert a == b
+@pytest.mark.parametrize(
+ ("args", "expected"),
+ [
+ pytest.param(
+ ({"x": 1, "y": 2},),
+ [("x", 1), ("y", 2)],
+ id="dict",
+ ),
+ pytest.param(
+ ({"x": literal(1), "y": literal(2)},),
+ [("x", 1), ("y", 2)],
+ id="dict_with_exprs",
+ ),
+ pytest.param(
+ ("x", 1, "y", 2),
+ [("x", 1), ("y", 2)],
+ id="variadic_pairs",
+ ),
+ pytest.param(
+ (literal("x"), literal(1), literal("y"), literal(2)),
+ [("x", 1), ("y", 2)],
+ id="variadic_with_exprs",
+ ),
+ ],
+)
+def test_make_map(args, expected):
+ ctx = SessionContext()
+ batch = pa.RecordBatch.from_arrays([pa.array([1])], names=["a"])
+ df = ctx.create_dataframe([[batch]])
+
+ result = df.select(f.make_map(*args).alias("m")).collect()[0].column(0)
+ assert result[0].as_py() == expected
+
+
+def test_make_map_from_two_lists():
+ ctx = SessionContext()
+ batch = pa.RecordBatch.from_arrays(
+ [
+ pa.array(["k1", "k2", "k3"]),
+ pa.array([10, 20, 30]),
+ ],
+ names=["keys", "vals"],
+ )
+ df = ctx.create_dataframe([[batch]])
+
+ m = f.make_map([column("keys")], [column("vals")])
+ result = df.select(f.map_keys(m).alias("k")).collect()[0].column(0)
+ assert result.to_pylist() == [["k1"], ["k2"], ["k3"]]
+
+ result = df.select(f.map_values(m).alias("v")).collect()[0].column(0)
+ assert result.to_pylist() == [[10], [20], [30]]
+
+
+def test_make_map_odd_args_raises():
+ with pytest.raises(ValueError, match="make_map expects"):
+ f.make_map("x", 1, "y")
+
+
+def test_make_map_mismatched_lengths():
+ with pytest.raises(ValueError, match="same length"):
+ f.make_map(["a", "b"], [1])
+
+
+@pytest.mark.parametrize(
+ ("func", "expected"),
+ [
+ pytest.param(f.map_keys, ["x", "y"], id="map_keys"),
+ pytest.param(f.map_values, [1, 2], id="map_values"),
+ pytest.param(
+ lambda m: f.map_extract(m, literal("x")),
+ [1],
+ id="map_extract",
+ ),
+ pytest.param(
+ lambda m: f.map_extract(m, literal("z")),
+ [None],
+ id="map_extract_missing_key",
+ ),
+ pytest.param(
+ f.map_entries,
+ [{"key": "x", "value": 1}, {"key": "y", "value": 2}],
+ id="map_entries",
+ ),
+ pytest.param(
+ lambda m: f.element_at(m, literal("y")),
+ [2],
+ id="element_at",
+ ),
+ ],
+)
+def test_map_functions(func, expected):
+ ctx = SessionContext()
+ batch = pa.RecordBatch.from_arrays([pa.array([1])], names=["a"])
+ df = ctx.create_dataframe([[batch]])
+
+ m = f.make_map({"x": 1, "y": 2})
+ result = df.select(func(m).alias("out")).collect()[0].column(0)
+ assert result[0].as_py() == expected
+
+
@pytest.mark.parametrize(
("function", "expected_result"),
[
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org