You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2023/01/30 12:12:57 UTC
[spark] branch branch-3.4 updated: [SPARK-41855][CONNECT][PYTHON][FOLLOWUP] Make `createDataFrame` accept `Decimal('NaN')`
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 545df6d3f89 [SPARK-41855][CONNECT][PYTHON][FOLLOWUP] Make `createDataFrame` accept `Decimal('NaN')`
545df6d3f89 is described below
commit 545df6d3f896270c6905d2a0bfbfaead36c3f8ea
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Mon Jan 30 21:12:36 2023 +0900
[SPARK-41855][CONNECT][PYTHON][FOLLOWUP] Make `createDataFrame` accept `Decimal('NaN')`
### What changes were proposed in this pull request?
Make `createDataFrame` accept `Decimal('NaN')`
### Why are the changes needed?
bugfix, refer to https://github.com/apache/spark/pull/34285
### Does this PR introduce _any_ user-facing change?
yes
### How was this patch tested?
enabled test
Closes #39800 from zhengruifeng/connect_decimal_nan.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
(cherry picked from commit fe507139b4a2e14f06a8bf2f792ebe6d8fe1fb3d)
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/sql/connect/conversion.py | 16 ++++++++++++++++
.../pyspark/sql/tests/connect/test_parity_dataframe.py | 5 -----
2 files changed, 16 insertions(+), 5 deletions(-)
diff --git a/python/pyspark/sql/connect/conversion.py b/python/pyspark/sql/connect/conversion.py
index 56712ae18f2..db647a9733a 100644
--- a/python/pyspark/sql/connect/conversion.py
+++ b/python/pyspark/sql/connect/conversion.py
@@ -16,6 +16,7 @@
#
import datetime
+import decimal
import pyarrow as pa
@@ -30,6 +31,7 @@ from pyspark.sql.types import (
ArrayType,
BinaryType,
NullType,
+ DecimalType,
)
from pyspark.sql.connect.types import to_arrow_schema
@@ -66,6 +68,9 @@ class LocalDataToArrowConversion:
elif isinstance(dataType, (TimestampType, TimestampNTZType)):
# Always truncate
return True
+ elif isinstance(dataType, DecimalType):
+ # Convert Decimal('NaN') to None
+ return True
else:
return False
@@ -168,6 +173,17 @@ class LocalDataToArrowConversion:
return convert_timestample
+ elif isinstance(dataType, DecimalType):
+
+ def convert_decimal(value: Any) -> Any:
+ if value is None:
+ return None
+ else:
+ assert isinstance(value, decimal.Decimal)
+ return None if value.is_nan() else value
+
+ return convert_decimal
+
else:
return lambda value: value
diff --git a/python/pyspark/sql/tests/connect/test_parity_dataframe.py b/python/pyspark/sql/tests/connect/test_parity_dataframe.py
index e04119bea9f..8e008cf9e1e 100644
--- a/python/pyspark/sql/tests/connect/test_parity_dataframe.py
+++ b/python/pyspark/sql/tests/connect/test_parity_dataframe.py
@@ -37,11 +37,6 @@ class DataFrameParityTests(DataFrameTestsMixin, ReusedConnectTestCase):
def test_create_dataframe_from_pandas_with_dst(self):
super().test_create_dataframe_from_pandas_with_dst()
- # TODO(SPARK-41855): createDataFrame doesn't handle None/NaN properly
- @unittest.skip("Fails in Spark Connect, should enable.")
- def test_create_nan_decimal_dataframe(self):
- super().test_create_nan_decimal_dataframe()
-
# TODO(SPARK-41870): Handle duplicate columns in `createDataFrame`
@unittest.skip("Fails in Spark Connect, should enable.")
def test_duplicated_column_names(self):
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org