You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2025/03/24 02:33:50 UTC
(spark) branch master updated: [SPARK-40353][PS][CONNECT] Fix index nullable mismatch in `ps.read_excel`
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new ae1997919417 [SPARK-40353][PS][CONNECT] Fix index nullable mismatch in `ps.read_excel`
ae1997919417 is described below
commit ae1997919417c5a1d9f32904be6d87985b13fe71
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Mon Mar 24 11:33:33 2025 +0900
[SPARK-40353][PS][CONNECT] Fix index nullable mismatch in `ps.read_excel`
### What changes were proposed in this pull request?
Fix nullable mismatch in `ps.read_excel`
### Why are the changes needed?
to re-enable the tests
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
updated ut
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #50323 from zhengruifeng/ps_read_excel.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/pandas/namespace.py | 30 ++++++++++--
.../pandas/tests/io/test_dataframe_spark_io.py | 53 ++++------------------
2 files changed, 33 insertions(+), 50 deletions(-)
diff --git a/python/pyspark/pandas/namespace.py b/python/pyspark/pandas/namespace.py
index d31bc1f48d11..a5c5a32037b8 100644
--- a/python/pyspark/pandas/namespace.py
+++ b/python/pyspark/pandas/namespace.py
@@ -65,6 +65,7 @@ from pyspark.sql.types import (
StringType,
DateType,
StructType,
+ StructField,
DataType,
)
from pyspark.sql.dataframe import DataFrame as PySparkDataFrame
@@ -85,6 +86,7 @@ from pyspark.pandas.utils import (
from pyspark.pandas.frame import DataFrame, _reduce_spark_multi
from pyspark.pandas.internal import (
InternalFrame,
+ InternalField,
DEFAULT_SERIES_NAME,
HIDDEN_COLUMNS,
SPARK_INDEX_NAME_FORMAT,
@@ -1186,9 +1188,28 @@ def read_excel(
pdf = pdf_or_pser
psdf = cast(DataFrame, from_pandas(pdf))
- return_schema = force_decimal_precision_scale(
- as_nullable_spark_type(psdf._internal.spark_frame.drop(*HIDDEN_COLUMNS).schema)
- )
+
+ raw_schema = psdf._internal.spark_frame.drop(*HIDDEN_COLUMNS).schema
+ index_scol_names = psdf._internal.index_spark_column_names
+ nullable_fields = []
+ for field in raw_schema.fields:
+ if field.name in index_scol_names:
+ nullable_fields.append(field)
+ else:
+ nullable_fields.append(
+ StructField(
+ field.name,
+ as_nullable_spark_type(field.dataType),
+ nullable=True,
+ metadata=field.metadata,
+ )
+ )
+ nullable_schema = StructType(nullable_fields)
+ return_schema = force_decimal_precision_scale(nullable_schema)
+
+ return_data_fields: Optional[List[InternalField]] = None
+ if psdf._internal.data_fields is not None:
+ return_data_fields = [f.normalize_spark_type() for f in psdf._internal.data_fields]
def output_func(pdf: pd.DataFrame) -> pd.DataFrame:
pdf = pd.concat([pd_read_excel(bin, sn=sn) for bin in pdf[pdf.columns[0]]])
@@ -1211,8 +1232,7 @@ def read_excel(
.select("content")
.mapInPandas(lambda iterator: map(output_func, iterator), schema=return_schema)
)
-
- return DataFrame(psdf._internal.with_new_sdf(sdf))
+ return DataFrame(psdf._internal.with_new_sdf(sdf, data_fields=return_data_fields))
if isinstance(pdf_or_psers, dict):
return {
diff --git a/python/pyspark/pandas/tests/io/test_dataframe_spark_io.py b/python/pyspark/pandas/tests/io/test_dataframe_spark_io.py
index b8225b10f1c7..74f152172e3d 100644
--- a/python/pyspark/pandas/tests/io/test_dataframe_spark_io.py
+++ b/python/pyspark/pandas/tests/io/test_dataframe_spark_io.py
@@ -253,8 +253,6 @@ class DataFrameSparkIOTestsMixin:
expected_idx.sort_values(by="f").to_spark().toPandas(),
)
- # TODO(SPARK-40353): re-enabling the `test_read_excel`.
- @unittest.skip("openpyxl")
def test_read_excel(self):
with self.temp_dir() as tmp:
path1 = "{}/file1.xlsx".format(tmp)
@@ -266,21 +264,22 @@ class DataFrameSparkIOTestsMixin:
pd.read_excel(open(path1, "rb"), index_col=0),
)
self.assert_eq(
- ps.read_excel(open(path1, "rb"), index_col=0, squeeze=True),
- pd.read_excel(open(path1, "rb"), index_col=0, squeeze=True),
+ ps.read_excel(open(path1, "rb"), index_col=0),
+ pd.read_excel(open(path1, "rb"), index_col=0),
)
self.assert_eq(ps.read_excel(path1), pd.read_excel(path1))
self.assert_eq(ps.read_excel(path1, index_col=0), pd.read_excel(path1, index_col=0))
self.assert_eq(
- ps.read_excel(path1, index_col=0, squeeze=True),
- pd.read_excel(path1, index_col=0, squeeze=True),
+ ps.read_excel(path1, index_col=0),
+ pd.read_excel(path1, index_col=0),
)
self.assert_eq(ps.read_excel(tmp), pd.read_excel(path1))
path2 = "{}/file2.xlsx".format(tmp)
self.test_pdf[["i32"]].to_excel(path2)
+ print(ps.read_excel(tmp, index_col=0).sort_index())
self.assert_eq(
ps.read_excel(tmp, index_col=0).sort_index(),
pd.concat(
@@ -288,11 +287,11 @@ class DataFrameSparkIOTestsMixin:
).sort_index(),
)
self.assert_eq(
- ps.read_excel(tmp, index_col=0, squeeze=True).sort_index(),
+ ps.read_excel(tmp, index_col=0).sort_index(),
pd.concat(
[
- pd.read_excel(path1, index_col=0, squeeze=True),
- pd.read_excel(path2, index_col=0, squeeze=True),
+ pd.read_excel(path1, index_col=0),
+ pd.read_excel(path2, index_col=0),
]
).sort_index(),
)
@@ -306,21 +305,12 @@ class DataFrameSparkIOTestsMixin:
sheet_names = [["Sheet_name_1", "Sheet_name_2"], None]
pdfs1 = pd.read_excel(open(path1, "rb"), sheet_name=None, index_col=0)
- pdfs1_squeezed = pd.read_excel(
- open(path1, "rb"), sheet_name=None, index_col=0, squeeze=True
- )
for sheet_name in sheet_names:
psdfs = ps.read_excel(open(path1, "rb"), sheet_name=sheet_name, index_col=0)
self.assert_eq(psdfs["Sheet_name_1"], pdfs1["Sheet_name_1"])
self.assert_eq(psdfs["Sheet_name_2"], pdfs1["Sheet_name_2"])
- psdfs = ps.read_excel(
- open(path1, "rb"), sheet_name=sheet_name, index_col=0, squeeze=True
- )
- self.assert_eq(psdfs["Sheet_name_1"], pdfs1_squeezed["Sheet_name_1"])
- self.assert_eq(psdfs["Sheet_name_2"], pdfs1_squeezed["Sheet_name_2"])
-
self.assert_eq(
ps.read_excel(tmp, index_col=0, sheet_name="Sheet_name_2"),
pdfs1["Sheet_name_2"],
@@ -331,30 +321,17 @@ class DataFrameSparkIOTestsMixin:
self.assert_eq(psdfs["Sheet_name_1"], pdfs1["Sheet_name_1"])
self.assert_eq(psdfs["Sheet_name_2"], pdfs1["Sheet_name_2"])
- psdfs = ps.read_excel(tmp, sheet_name=sheet_name, index_col=0, squeeze=True)
- self.assert_eq(psdfs["Sheet_name_1"], pdfs1_squeezed["Sheet_name_1"])
- self.assert_eq(psdfs["Sheet_name_2"], pdfs1_squeezed["Sheet_name_2"])
-
path2 = "{}/file2.xlsx".format(tmp)
with pd.ExcelWriter(path2) as writer:
self.test_pdf.to_excel(writer, sheet_name="Sheet_name_1")
self.test_pdf[["i32"]].to_excel(writer, sheet_name="Sheet_name_2")
pdfs2 = pd.read_excel(path2, sheet_name=None, index_col=0)
- pdfs2_squeezed = pd.read_excel(path2, sheet_name=None, index_col=0, squeeze=True)
self.assert_eq(
ps.read_excel(tmp, sheet_name="Sheet_name_2", index_col=0).sort_index(),
pd.concat([pdfs1["Sheet_name_2"], pdfs2["Sheet_name_2"]]).sort_index(),
)
- self.assert_eq(
- ps.read_excel(
- tmp, sheet_name="Sheet_name_2", index_col=0, squeeze=True
- ).sort_index(),
- pd.concat(
- [pdfs1_squeezed["Sheet_name_2"], pdfs2_squeezed["Sheet_name_2"]]
- ).sort_index(),
- )
for sheet_name in sheet_names:
psdfs = ps.read_excel(tmp, sheet_name=sheet_name, index_col=0)
@@ -367,20 +344,6 @@ class DataFrameSparkIOTestsMixin:
pd.concat([pdfs1["Sheet_name_2"], pdfs2["Sheet_name_2"]]).sort_index(),
)
- psdfs = ps.read_excel(tmp, sheet_name=sheet_name, index_col=0, squeeze=True)
- self.assert_eq(
- psdfs["Sheet_name_1"].sort_index(),
- pd.concat(
- [pdfs1_squeezed["Sheet_name_1"], pdfs2_squeezed["Sheet_name_1"]]
- ).sort_index(),
- )
- self.assert_eq(
- psdfs["Sheet_name_2"].sort_index(),
- pd.concat(
- [pdfs1_squeezed["Sheet_name_2"], pdfs2_squeezed["Sheet_name_2"]]
- ).sort_index(),
- )
-
def test_read_orc(self):
with self.temp_dir() as tmp:
path = "{}/file1.orc".format(tmp)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org