You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2026/08/21 00:53:31 UTC
(spark) branch master updated: [SPARK-58919][PYTHON][TEST] Add tests for pa.Schema.from_pandas
This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 56132f95cec5 [SPARK-58919][PYTHON][TEST] Add tests for pa.Schema.from_pandas
56132f95cec5 is described below
commit 56132f95cec5e25427acde5baa39bb44e91edbd2
Author: Spenser Sun <hs...@gmail.com>
AuthorDate: Fri Aug 21 08:53:19 2026 +0800
[SPARK-58919][PYTHON][TEST] Add tests for pa.Schema.from_pandas
### What changes were proposed in this pull request?
Add golden-file tests for `pa.Schema.from_pandas` in `test_pyarrow_dataframe_from_pandas.py`, alongside the existing `RecordBatch.from_pandas` tests. The test sweeps `preserve_index` (`None`/`False`/`True`) across frame shape and index kind (0-column, single/multi-column, RangeIndex, named, unnamed, and MultiIndex), pinning each inferred field's name, type, and nullability.
The index-aware input rendering (`_index_desc`/`_input_cell`) is lifted from the RecordBatch class into the shared base (now a second consumer), and `GoldenFileTestMixin` gains a `pa.Schema` repr (`repr_arrow_schema_value`) that surfaces field nullability.
### Why are the changes needed?
Part of SPARK-54936 (monitor upstream library behavior changes). Spark builds a `StructType` from `pa.Schema.from_pandas` at two sites that diverge on `preserve_index`: classic `createDataFrame` passes `preserve_index=False` (index dropped), while Spark Connect passes it bare/`None` (a named or non-range index becomes an extra field). Both read each field's type and nullability. Pinning this in a golden file catches an upstream behavior change loudly instead of silently altering the i [...]
### Does this PR introduce _any_ user-facing change?
No. Test-only.
### How was this patch tested?
New golden-file tests, run without a Spark session. Validated across a PyArrow 18-25 x pandas 2/3 sweep (16 combinations) with no version-specific overrides.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Opus 4.8)
Closes #58178 from Spenserrrr/schema-from-pandas-tests.
Authored-by: Spenser Sun <hs...@gmail.com>
Signed-off-by: Ruifeng Zheng <ru...@foxmail.com>
---
python/pyspark/testing/goldenutils.py | 24 +++++
.../pyarrow/golden_pyarrow_schema_from_pandas.csv | 13 +++
.../pyarrow/golden_pyarrow_schema_from_pandas.md | 14 +++
.../pyarrow/test_pyarrow_dataframe_from_pandas.py | 106 +++++++++++++++++----
4 files changed, 141 insertions(+), 16 deletions(-)
diff --git a/python/pyspark/testing/goldenutils.py b/python/pyspark/testing/goldenutils.py
index 8374d20eb422..df699a81b065 100644
--- a/python/pyspark/testing/goldenutils.py
+++ b/python/pyspark/testing/goldenutils.py
@@ -408,6 +408,27 @@ class GoldenFileTestMixin:
v_str, schema = cls._repr_arrow_columns(value, max_len)
return f"{v_str}@RecordBatch[{schema}]"
+ @classmethod
+ def repr_arrow_schema_value(cls, value: "pa.Schema", max_len: int = 32) -> str:
+ """
+ Format a PyArrow Schema for golden file.
+
+ Renders each field as "name: type nullable=...". A Schema carries no data, so
+ nullability is included (unlike the Table/RecordBatch schema string): Spark reads
+ ``field.nullable`` to build its StructType, so a change there silently alters the
+ inferred Spark schema.
+
+ Returns
+ -------
+ str
+ "[name: type nullable=True, ...]@Schema"
+ """
+ fields = [f"{f.name}: {cls.repr_type(f.type)} nullable={f.nullable}" for f in value]
+ v_str = "[" + ", ".join(fields) + "]"
+ if max_len > 0:
+ v_str = v_str[:max_len]
+ return f"{v_str}@Schema"
+
@classmethod
def repr_pandas_value(cls, value: "pd.DataFrame", max_len: int = 32) -> str:
"""
@@ -483,6 +504,7 @@ class GoldenFileTestMixin:
- PyArrow Array/ChunkedArray -> repr_arrow_value
- PyArrow Table -> repr_arrow_table_value
- PyArrow RecordBatch -> repr_arrow_record_batch_value
+ - PyArrow Schema -> repr_arrow_schema_value
- pandas DataFrame -> repr_pandas_value
- numpy ndarray -> repr_numpy_value
- Everything else -> repr_python_value
@@ -505,6 +527,8 @@ class GoldenFileTestMixin:
return cls.repr_arrow_table_value(value, max_len)
if have_pyarrow and isinstance(value, pa.RecordBatch):
return cls.repr_arrow_record_batch_value(value, max_len)
+ if have_pyarrow and isinstance(value, pa.Schema):
+ return cls.repr_arrow_schema_value(value, max_len)
if have_pandas and isinstance(value, pd.DataFrame):
return cls.repr_pandas_value(value, max_len)
diff --git a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_schema_from_pandas.csv b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_schema_from_pandas.csv
new file mode 100644
index 000000000000..66f7bbaf691b
--- /dev/null
+++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_schema_from_pandas.csv
@@ -0,0 +1,13 @@
+test case pandas dataframe preserve_index=None preserve_index=False preserve_index=True
+0-columns:range-index {}@Dataframe[][index=RangeIndex[0:3:1]] []@Schema []@Schema [__index_level_0__: int64 nullable=True]@Schema
+0-columns:named-index {}@Dataframe[][index='idx':[100, 200, 300]] [idx: int64 nullable=True]@Schema []@Schema [idx: int64 nullable=True]@Schema
+0-columns:unnamed-index {}@Dataframe[][index=None:[10, 20, 30]] [__index_level_0__: int64 nullable=True]@Schema []@Schema [__index_level_0__: int64 nullable=True]@Schema
+0-columns:empty {}@Dataframe[][index=RangeIndex[0:0:1]] []@Schema []@Schema [__index_level_0__: int64 nullable=True]@Schema
+single-column:range-index {'a': [1, 2, 3]}@Dataframe[a int64][index=RangeIndex[0:3:1]] [a: int64 nullable=True]@Schema [a: int64 nullable=True]@Schema [a: int64 nullable=True, __index_level_0__: int64 nullable=True]@Schema
+single-column:named-index {'a': [1, 2, 3]}@Dataframe[a int64][index='idx':[100, 200, 300]] [a: int64 nullable=True, idx: int64 nullable=True]@Schema [a: int64 nullable=True]@Schema [a: int64 nullable=True, idx: int64 nullable=True]@Schema
+single-column:unnamed-index {'a': [1, 2, 3]}@Dataframe[a int64][index=None:[10, 20, 30]] [a: int64 nullable=True, __index_level_0__: int64 nullable=True]@Schema [a: int64 nullable=True]@Schema [a: int64 nullable=True, __index_level_0__: int64 nullable=True]@Schema
+multi-column:standard {'i': [1, 2, 3], 'f': [1.5, 2.5, 3.5], 'b': [True, False, True], 's': ['a', 'b', 'c'], 't': [Timestamp('2020-01-01 05:30:00'), Timestamp('2020-01-01 05:30:00'), Timestamp('2020-01-01 05:30:00')]}@Dataframe[i int64, f float64, b bool, s object, t datetime64[ns]][index=RangeIndex[0:3:1]] [i: int64 nullable=True, f: float64 nullable=True, b: bool nullable=True, s: string nullable=True, t: timestamp[ns] nullable=True]@Schema [i: int64 nullable=True, f: float64 nullable= [...]
+multi-column:nullable {'f': [1.5, nan, 3.5], 'b': [True, None, False], 's': ['a', None, 'c'], 't': [Timestamp('2020-01-01 05:30:00'), NaT, Timestamp('2020-01-01 05:30:00')]}@Dataframe[f float64, b object, s object, t datetime64[ns]][index=RangeIndex[0:3:1]] [f: float64 nullable=True, b: bool nullable=True, s: string nullable=True, t: timestamp[ns] nullable=True]@Schema [f: float64 nullable=True, b: bool nullable=True, s: string nullable=True, t: timestamp[ns] nullable=True]@Schema [f: fl [...]
+multi-column:no-rows {'i': [], 'f': [], 'b': [], 't': []}@Dataframe[i int64, f float64, b bool, t datetime64[ns]][index=RangeIndex[0:0:1]] [i: int64 nullable=True, f: float64 nullable=True, b: bool nullable=True, t: timestamp[ns] nullable=True]@Schema [i: int64 nullable=True, f: float64 nullable=True, b: bool nullable=True, t: timestamp[ns] nullable=True]@Schema [i: int64 nullable=True, f: float64 nullable=True, b: bool nullable=True, t: timestamp[ns] nullable=True, __index_level_0__: in [...]
+single-column:multiindex {'a': [1, 2, 3]}@Dataframe[a int64][index=MultiIndex[names=['g', 'n']]] [a: int64 nullable=True, g: int64 nullable=True, n: int64 nullable=True]@Schema [a: int64 nullable=True]@Schema [a: int64 nullable=True, g: int64 nullable=True, n: int64 nullable=True]@Schema
+single-column:multiindex-partial-name {'a': [1, 2]}@Dataframe[a int64][index=MultiIndex[names=['g', None]]] [a: int64 nullable=True, g: int64 nullable=True, __index_level_1__: int64 nullable=True]@Schema [a: int64 nullable=True]@Schema [a: int64 nullable=True, g: int64 nullable=True, __index_level_1__: int64 nullable=True]@Schema
diff --git a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_schema_from_pandas.md b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_schema_from_pandas.md
new file mode 100644
index 000000000000..bcf6286bdae4
--- /dev/null
+++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_schema_from_pandas.md
@@ -0,0 +1,14 @@
+| test case | pandas dataframe | preserve_index=None | preserve_index=False [...]
+|---------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|---------------------- [...]
+| 0-columns:range-index | {}@Dataframe[][index=RangeIndex[0:3:1]] | []@Schema | []@Schema [...]
+| 0-columns:named-index | {}@Dataframe[][index='idx':[100, 200, 300]] | [idx: int64 nullable=True]@Schema | []@Schema [...]
+| 0-columns:unnamed-index | {}@Dataframe[][index=None:[10, 20, 30]] | [__index_level_0__: int64 nullable=True]@Schema | []@Schema [...]
+| 0-columns:empty | {}@Dataframe[][index=RangeIndex[0:0:1]] | []@Schema | []@Schema [...]
+| single-column:range-index | {'a': [1, 2, 3]}@Dataframe[a int64][index=RangeIndex[0:3:1]] | [a: int64 nullable=True]@Schema | [a: int64 nullable=Tr [...]
+| single-column:named-index | {'a': [1, 2, 3]}@Dataframe[a int64][index='idx':[100, 200, 300]] | [a: int64 nullable=True, idx: int64 nullable=True]@Schema | [a: int64 nullable=Tr [...]
+| single-column:unnamed-index | {'a': [1, 2, 3]}@Dataframe[a int64][index=None:[10, 20, 30]] | [a: int64 nullable=True, __index_level_0__: int64 nullable=True]@Schema | [a: int64 nullable=Tr [...]
+| multi-column:standard | {'i': [1, 2, 3], 'f': [1.5, 2.5, 3.5], 'b': [True, False, True], 's': ['a', 'b', 'c'], 't': [Timestamp('2020-01-01 05:30:00'), Timestamp('2020-01-01 05:30:00'), Timestamp('2020-01-01 05:30:00')]}@Dataframe[i int64, f float64, b bool, s object, t datetime64[ns]][index=RangeIndex[0:3:1]] | [i: int64 nullable=True, f: float64 nullable=True, b: bool nullable=True, s: string nullable=True, t: timestamp[ns] nullable=True]@Schema | [i: int64 nullable=Tr [...]
+| multi-column:nullable | {'f': [1.5, nan, 3.5], 'b': [True, None, False], 's': ['a', None, 'c'], 't': [Timestamp('2020-01-01 05:30:00'), NaT, Timestamp('2020-01-01 05:30:00')]}@Dataframe[f float64, b object, s object, t datetime64[ns]][index=RangeIndex[0:3:1]] | [f: float64 nullable=True, b: bool nullable=True, s: string nullable=True, t: timestamp[ns] nullable=True]@Schema | [f: float64 nullable= [...]
+| multi-column:no-rows | {'i': [], 'f': [], 'b': [], 't': []}@Dataframe[i int64, f float64, b bool, t datetime64[ns]][index=RangeIndex[0:0:1]] | [i: int64 nullable=True, f: float64 nullable=True, b: bool nullable=True, t: timestamp[ns] nullable=True]@Schema | [i: int64 nullable=Tr [...]
+| single-column:multiindex | {'a': [1, 2, 3]}@Dataframe[a int64][index=MultiIndex[names=['g', 'n']]] | [a: int64 nullable=True, g: int64 nullable=True, n: int64 nullable=True]@Schema | [a: int64 nullable=Tr [...]
+| single-column:multiindex-partial-name | {'a': [1, 2]}@Dataframe[a int64][index=MultiIndex[names=['g', None]]] | [a: int64 nullable=True, g: int64 nullable=True, __index_level_1__: int64 nullable=True]@Schema | [a: int64 nullable=Tr [...]
\ No newline at end of file
diff --git a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_dataframe_from_pandas.py b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_dataframe_from_pandas.py
index 0b7af403fa07..b125e25d92d8 100644
--- a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_dataframe_from_pandas.py
+++ b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_dataframe_from_pandas.py
@@ -17,8 +17,8 @@
"""
Golden-file tests for the PyArrow ``from_pandas`` constructors that take a whole pandas
-DataFrame: ``pa.RecordBatch.from_pandas`` (with ``pa.Table.from_pandas`` /
-``pa.Schema.from_pandas`` to follow in this file). These take a DataFrame, unlike
+DataFrame: ``pa.RecordBatch.from_pandas`` and ``pa.Schema.from_pandas`` (with
+``pa.Table.from_pandas`` to follow in this file). These take a DataFrame, unlike
``pa.Array.from_pandas`` which takes a Series (covered by test_pyarrow_array_from_pandas_*).
Per-column type inference matches the Array tests, so these pin the DataFrame-level
@@ -29,6 +29,13 @@ for RecordBatch -- num_rows preservation for a 0-column DataFrame. Spark calls
the default ``preserve_index=None`` to carry num_rows via the index metadata -- otherwise a
0-column relation loses its rows.
+``Schema.from_pandas`` is inspected to build a Spark schema, and the two prod call sites
+diverge on ``preserve_index``: classic pandas/conversion.py:971 passes ``False`` (index
+dropped), Connect session.py:573 passes it bare/``None`` (a named or non-range index becomes
+an extra field) -- so a named-index frame yields different field sets. Spark reads each
+field's type AND nullability (conversion.py:989 / session.py:590), so the schema test pins
+name/type/nullability across ``preserve_index``.
+
Regenerate with SPARK_GENERATE_GOLDEN_FILES=1.
"""
@@ -51,11 +58,25 @@ if have_pyarrow:
class _PyArrowFromPandasFrameTestBase(GoldenFileTestMixin, unittest.TestCase):
"""
- Shared machinery for the DataFrame-input ``from_pandas`` constructors (RecordBatch
- here; Table/Schema as followups). Owns the source-frame inventory and defines no
- ``test_*`` of its own.
+ Shared machinery for the DataFrame-input ``from_pandas`` constructors (RecordBatch and
+ Schema here; Table as a followup). Owns the source-frame inventory and the index-aware
+ input-cell rendering (both constructors depend on the index under ``preserve_index``);
+ defines no ``test_*`` of its own.
"""
+ @staticmethod
+ def _index_desc(index) -> str:
+ """Compact, deterministic description of a pandas index for the input cell."""
+ if isinstance(index, pd.MultiIndex):
+ return f"MultiIndex[names={list(index.names)}]"
+ if isinstance(index, pd.RangeIndex):
+ return f"RangeIndex[{index.start}:{index.stop}:{index.step}]"
+ return f"{index.name!r}:{index.tolist()}"
+
+ def _input_cell(self, df) -> str:
+ """Input DataFrame repr, extended with its index (repr_value drops it)."""
+ return f"{self.repr_value(df, max_len=0)}[index={self._index_desc(df.index)}]"
+
def _build_source_frames(self):
"""Named pandas DataFrames covering shape x index-kind, plus a dtype sample."""
dt = datetime.datetime(2020, 1, 1, 5, 30)
@@ -120,17 +141,6 @@ class _PyArrowFromPandasFrameTestBase(GoldenFileTestMixin, unittest.TestCase):
class PyArrowRecordBatchFromPandasTests(_PyArrowFromPandasFrameTestBase):
"""Tests pa.RecordBatch.from_pandas() across preserve_index via golden file comparison."""
- @staticmethod
- def _index_desc(index) -> str:
- """Compact, deterministic description of a pandas index for the input cell."""
- if isinstance(index, pd.RangeIndex):
- return f"RangeIndex[{index.start}:{index.stop}:{index.step}]"
- return f"{index.name!r}:{index.tolist()}"
-
- def _input_cell(self, df) -> str:
- """Input DataFrame repr, extended with its index (repr_value drops it)."""
- return f"{self.repr_value(df, max_len=0)}[index={self._index_desc(df.index)}]"
-
def _from_pandas_cell(self, df, **kwargs) -> str:
"""
Convert ``df`` via RecordBatch.from_pandas(**kwargs) and append num_rows -- the
@@ -173,6 +183,70 @@ class PyArrowRecordBatchFromPandasTests(_PyArrowFromPandasFrameTestBase):
)
+@unittest.skipIf(
+ not have_pyarrow or not have_pandas,
+ pyarrow_requirement_message or pandas_requirement_message,
+)
+class PyArrowSchemaFromPandasTests(_PyArrowFromPandasFrameTestBase):
+ """Tests pa.Schema.from_pandas() across preserve_index via golden file comparison."""
+
+ def _schema_source_frames(self):
+ """Shared frames plus two MultiIndex rows -- a MultiIndex has several index levels,
+ each becoming its own field, so these pin multi-level index-to-field naming at the
+ schema. Level values are integers (stable int64 on pandas 2 and 3; strings drift)."""
+ frames = self._build_source_frames()
+ frames["single-column:multiindex"] = pd.DataFrame(
+ {"a": [1, 2, 3]},
+ index=pd.MultiIndex.from_tuples([(1, 10), (1, 20), (2, 30)], names=["g", "n"]),
+ )
+ frames["single-column:multiindex-partial-name"] = pd.DataFrame(
+ {"a": [1, 2]},
+ index=pd.MultiIndex.from_tuples([(1, 10), (2, 20)], names=["g", None]),
+ )
+ return frames
+
+ def _from_pandas_cell(self, df, **kwargs) -> str:
+ """
+ Infer the schema via Schema.from_pandas(**kwargs) and render its fields with
+ nullability -- the name/type/nullable Spark reads to build its StructType. Returns
+ ERR@<ExceptionClass> if inference raises; a formatting error is a test bug.
+ """
+ try:
+ schema = pa.Schema.from_pandas(df, **kwargs)
+ except Exception as e:
+ return f"ERR@{type(e).__name__}"
+ return self.repr_value(schema, max_len=0)
+
+ def test_from_pandas(self):
+ """Test pa.Schema.from_pandas() across preserve_index against golden file."""
+ sources = self._schema_source_frames()
+ row_names = list(sources.keys())
+ preserve = {
+ "preserve_index=None": None,
+ "preserve_index=False": False,
+ "preserve_index=True": True,
+ }
+ col_names = ["pandas dataframe", *preserve.keys()]
+
+ # Version-specific expected values go here, keyed by (row, col), for known drift.
+ overrides: dict[tuple[str, str], str] = {}
+
+ def compute_cell(row_name, col_name):
+ df = sources[row_name]
+ if col_name == "pandas dataframe":
+ return self._input_cell(df)
+ return self._from_pandas_cell(df, preserve_index=preserve[col_name])
+
+ self.compare_or_generate_golden_matrix(
+ row_names=row_names,
+ col_names=col_names,
+ compute_cell=compute_cell,
+ golden_file_prefix="golden_pyarrow_schema_from_pandas",
+ index_name="test case",
+ overrides=overrides,
+ )
+
+
if __name__ == "__main__":
from pyspark.testing import main
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org