You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2024/05/13 08:17:54 UTC
(spark) branch branch-3.5 updated: [SPARK-48248][PYTHON] Fix nested array to respect legacy conf of inferArrayTypeFromFirstElement
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new ab511a784db2 [SPARK-48248][PYTHON] Fix nested array to respect legacy conf of inferArrayTypeFromFirstElement
ab511a784db2 is described below
commit ab511a784db2d2b9d0980b63a02fea8f472ceb76
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Mon May 13 17:15:28 2024 +0900
[SPARK-48248][PYTHON] Fix nested array to respect legacy conf of inferArrayTypeFromFirstElement
This PR fixes a bug that does not respect `spark.sql.pyspark.legacy.inferArrayTypeFromFirstElement.enabled` in nested arrays, introduced by https://github.com/apache/spark/pull/36545.
To have a way to restore the original behaviour.
Yes, it fixes the regression when `spark.sql.pyspark.legacy.inferArrayTypeFromFirstElement.enabled` is set to `True`.
Unittest added.
No.
Closes #46548 from HyukjinKwon/SPARK-48248.
Authored-by: Hyukjin Kwon <gu...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
(cherry picked from commit b2140d0f25d81e64a968df83c5da5089051acaac)
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/sql/tests/test_types.py | 7 +++++++
python/pyspark/sql/types.py | 18 ++++++++++++++++--
2 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/python/pyspark/sql/tests/test_types.py b/python/pyspark/sql/tests/test_types.py
index 00bd1d9a6f83..ad458988d4fd 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -1275,6 +1275,13 @@ class TypesTestsMixin:
schema3 = self.spark.sql("SELECT INTERVAL '8' MONTH AS interval").schema
self.assertEqual(schema3.fields[0].dataType, YearMonthIntervalType(1, 1))
+ def test_infer_array_element_type_with_struct(self):
+ # SPARK-48248: Nested array to respect legacy conf of inferArrayTypeFromFirstElement
+ with self.sql_conf(
+ {"spark.sql.pyspark.legacy.inferArrayTypeFromFirstElement.enabled": True}
+ ):
+ self.assertEqual([[1, None]], self.spark.createDataFrame([[[[1, "a"]]]]).first()[0])
+
class DataTypeTests(unittest.TestCase):
# regression test for SPARK-6055
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 24964c56e2e8..a2a879695762 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -1606,13 +1606,27 @@ def _infer_type(
if len(obj) > 0:
if infer_array_from_first_element:
return ArrayType(
- _infer_type(obj[0], infer_dict_as_struct, prefer_timestamp_ntz), True
+ _infer_type(
+ obj[0],
+ infer_dict_as_struct,
+ infer_array_from_first_element,
+ prefer_timestamp_ntz,
+ ),
+ True,
)
else:
return ArrayType(
reduce(
_merge_type,
- (_infer_type(v, infer_dict_as_struct, prefer_timestamp_ntz) for v in obj),
+ (
+ _infer_type(
+ v,
+ infer_dict_as_struct,
+ infer_array_from_first_element,
+ prefer_timestamp_ntz,
+ )
+ for v in obj
+ ),
),
True,
)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org