You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2022/12/22 07:11:18 UTC
[spark] branch master updated: [SPARK-41675][CONNECT][PYTHON] Make Column op support `datetime`
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e56f31d5e76 [SPARK-41675][CONNECT][PYTHON] Make Column op support `datetime`
e56f31d5e76 is described below
commit e56f31d5e764cd68afed87675e63be7f4fff3e54
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Thu Dec 22 16:10:58 2022 +0900
[SPARK-41675][CONNECT][PYTHON] Make Column op support `datetime`
### What changes were proposed in this pull request?
Make column op support `datetime`
### Why are the changes needed?
should support it like the pyspark
### Does this PR introduce _any_ user-facing change?
yes
### How was this patch tested?
added ut
Closes #39171 from zhengruifeng/connect_column_datatypes.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/sql/connect/column.py | 7 +--
.../sql/tests/connect/test_connect_column.py | 61 ++++++++++++++++++++++
2 files changed, 63 insertions(+), 5 deletions(-)
diff --git a/python/pyspark/sql/connect/column.py b/python/pyspark/sql/connect/column.py
index 07b435e917b..b43b776e745 100644
--- a/python/pyspark/sql/connect/column.py
+++ b/python/pyspark/sql/connect/column.py
@@ -16,7 +16,6 @@
#
from typing import (
- get_args,
TYPE_CHECKING,
Callable,
Any,
@@ -82,10 +81,9 @@ def _bin_op(
name: str, doc: Optional[str] = "binary function", reverse: bool = False
) -> Callable[["Column", Any], "Column"]:
def wrapped(self: "Column", other: Any) -> "Column":
- from pyspark.sql.connect._typing import PrimitiveType
from pyspark.sql.connect.functions import lit
- if isinstance(other, get_args(PrimitiveType)):
+ if isinstance(other, (bool, float, int, str, datetime.datetime, datetime.date)):
other = lit(other)
if not reverse:
return scalar_function(name, self, other)
@@ -781,10 +779,9 @@ class Column:
"""Returns a binary expression with the current column as the left
side and the other expression as the right side.
"""
- from pyspark.sql.connect._typing import PrimitiveType
from pyspark.sql.connect.functions import lit
- if isinstance(other, get_args(PrimitiveType)):
+ if isinstance(other, (bool, float, int, str, datetime.datetime, datetime.date)):
other = lit(other)
return scalar_function("==", self, other)
diff --git a/python/pyspark/sql/tests/connect/test_connect_column.py b/python/pyspark/sql/tests/connect/test_connect_column.py
index be5848c5075..ae47a5ec006 100644
--- a/python/pyspark/sql/tests/connect/test_connect_column.py
+++ b/python/pyspark/sql/tests/connect/test_connect_column.py
@@ -108,6 +108,67 @@ class SparkConnectTests(SparkConnectSQLTestCase):
df4.filter(df4.name.isNotNull()).toPandas(),
)
+ def test_datetime(self):
+ query = """
+ SELECT * FROM VALUES
+ (TIMESTAMP('2022-12-22 15:50:00'), DATE('2022-12-25'), 1.1),
+ (TIMESTAMP('2022-12-22 18:50:00'), NULL, 2.2),
+ (TIMESTAMP('2022-12-23 15:50:00'), DATE('2022-12-24'), 3.3),
+ (NULL, DATE('2022-12-22'), NULL)
+ AS tab(a, b, c)
+ """
+ # +-------------------+----------+----+
+ # | a| b| c|
+ # +-------------------+----------+----+
+ # |2022-12-22 15:50:00|2022-12-25| 1.1|
+ # |2022-12-22 18:50:00| null| 2.2|
+ # |2022-12-23 15:50:00|2022-12-24| 3.3|
+ # | null|2022-12-22|null|
+ # +-------------------+----------+----+
+
+ cdf = self.spark.sql(query)
+ sdf = self.connect.sql(query)
+
+ # datetime.date
+ self.assert_eq(
+ cdf.select(cdf.a < datetime.date(2022, 12, 23)).toPandas(),
+ sdf.select(sdf.a < datetime.date(2022, 12, 23)).toPandas(),
+ )
+ self.assert_eq(
+ cdf.select(cdf.a != datetime.date(2022, 12, 23)).toPandas(),
+ sdf.select(sdf.a != datetime.date(2022, 12, 23)).toPandas(),
+ )
+ self.assert_eq(
+ cdf.select(cdf.a == datetime.date(2022, 12, 22)).toPandas(),
+ sdf.select(sdf.a == datetime.date(2022, 12, 22)).toPandas(),
+ )
+ self.assert_eq(
+ cdf.select(cdf.b < datetime.date(2022, 12, 23)).toPandas(),
+ sdf.select(sdf.b < datetime.date(2022, 12, 23)).toPandas(),
+ )
+ self.assert_eq(
+ cdf.select(cdf.b >= datetime.date(2022, 12, 23)).toPandas(),
+ sdf.select(sdf.b >= datetime.date(2022, 12, 23)).toPandas(),
+ )
+
+ # datetime.datetime
+ self.assert_eq(
+ cdf.select(cdf.a < datetime.datetime(2022, 12, 22, 17, 0, 0)).toPandas(),
+ sdf.select(sdf.a < datetime.datetime(2022, 12, 22, 17, 0, 0)).toPandas(),
+ )
+ self.assert_eq(
+ cdf.select(cdf.a > datetime.datetime(2022, 12, 22, 17, 0, 0)).toPandas(),
+ sdf.select(sdf.a > datetime.datetime(2022, 12, 22, 17, 0, 0)).toPandas(),
+ )
+ self.assert_eq(
+ cdf.select(cdf.b >= datetime.datetime(2022, 12, 23, 17, 0, 0)).toPandas(),
+ sdf.select(sdf.b >= datetime.datetime(2022, 12, 23, 17, 0, 0)).toPandas(),
+ )
+ self.assert_eq(
+ cdf.select(cdf.b < datetime.datetime(2022, 12, 23, 17, 0, 0)).toPandas(),
+ sdf.select(sdf.b < datetime.datetime(2022, 12, 23, 17, 0, 0)).toPandas(),
+ )
+
def test_simple_binary_expressions(self):
"""Test complex expression"""
df = self.connect.read.table(self.tbl_name)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org