You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ue...@apache.org on 2021/06/29 02:04:29 UTC
[spark] branch master updated: [SPARK-35344][PYTHON] Support
creating a Column of numpy literals in pandas API on Spark
This is an automated email from the ASF dual-hosted git repository.
ueshin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5f0113e [SPARK-35344][PYTHON] Support creating a Column of numpy literals in pandas API on Spark
5f0113e is described below
commit 5f0113e3a666f64c8b37cc889af846299a84960d
Author: Xinrong Meng <xi...@databricks.com>
AuthorDate: Mon Jun 28 19:03:42 2021 -0700
[SPARK-35344][PYTHON] Support creating a Column of numpy literals in pandas API on Spark
### What changes were proposed in this pull request?
The PR is proposed to support creating a Column of numpy literal value in pandas-on-Spark. It consists of three changes mainly:
- Enable the `lit` function defined in `pyspark.pandas.spark.functions` to support numpy literals input.
```py
>>> from pyspark.pandas.spark import functions as SF
>>> SF.lit(np.int64(1))
Column<'CAST(1 AS BIGINT)'>
>>> SF.lit(np.int32(1))
Column<'CAST(1 AS INT)'>
>>> SF.lit(np.int8(1))
Column<'CAST(1 AS TINYINT)'>
>>> SF.lit(np.byte(1))
Column<'CAST(1 AS TINYINT)'>
>>> SF.lit(np.float32(1))
Column<'CAST(1.0 AS FLOAT)'>
```
- Substitute `F.lit` by `SF.lit`, that is, use `lit` function defined in `pyspark.pandas.spark.functions` rather than `lit` function defined in `pyspark.sql.functions` to allow creating columns out of numpy literals.
- Enable numpy literals input in `isin` method
Non-goal:
- Some pandas-on-Spark APIs use PySpark column-related APIs internally, and these column-related APIs don't support numpy literals, thus numpy literals are disallowed as input (e.g. `to_replace` parameter in `replace` API). This PR doesn't aim to adjust all of them. This PR adjusts `isin` only, because the PR is inspired by that (as https://github.com/databricks/koalas/issues/2161).
- To complete mappings between all kinds of numpy literals and Spark data types should be a followup task.
### Why are the changes needed?
Spark (`lit` function defined in `pyspark.sql.functions`) doesn't support creating a Column out of numpy literal value.
So `lit` function defined in `pyspark.pandas.spark.functions` is adjusted in order to support that in pandas-on-Spark.
### Does this PR introduce _any_ user-facing change?
Yes.
Before:
```py
>>> a = ps.DataFrame({'source': [1,2,3,4,5]})
>>> a.source.isin([np.int64(1), np.int64(2)])
Traceback (most recent call last):
...
AttributeError: 'numpy.int64' object has no attribute '_get_object_id'
```
After:
```py
>>> a = ps.DataFrame({'source': [1,2,3,4,5]})
>>> a.source.isin([np.int64(1), np.int64(2)])
0 True
1 True
2 False
3 False
4 False
Name: source, dtype: bool
```
### How was this patch tested?
Unit tests.
Closes #32955 from xinrong-databricks/datatypeops_literal.
Authored-by: Xinrong Meng <xi...@databricks.com>
Signed-off-by: Takuya UESHIN <ue...@databricks.com>
---
dev/sparktestsupport/modules.py | 1 +
python/pyspark/pandas/base.py | 25 ++++----
python/pyspark/pandas/data_type_ops/base.py | 9 +--
python/pyspark/pandas/data_type_ops/binary_ops.py | 5 +-
python/pyspark/pandas/data_type_ops/boolean_ops.py | 17 ++---
.../pandas/data_type_ops/categorical_ops.py | 5 +-
python/pyspark/pandas/data_type_ops/date_ops.py | 5 +-
.../pyspark/pandas/data_type_ops/datetime_ops.py | 5 +-
python/pyspark/pandas/data_type_ops/num_ops.py | 50 ++++++++-------
python/pyspark/pandas/data_type_ops/string_ops.py | 6 +-
python/pyspark/pandas/frame.py | 75 +++++++++++-----------
python/pyspark/pandas/generic.py | 10 +--
python/pyspark/pandas/groupby.py | 11 ++--
python/pyspark/pandas/indexes/base.py | 7 +-
python/pyspark/pandas/indexes/multi.py | 9 +--
python/pyspark/pandas/indexing.py | 67 ++++++++++---------
python/pyspark/pandas/namespace.py | 3 +-
python/pyspark/pandas/numpy_compat.py | 3 +-
python/pyspark/pandas/plot/core.py | 5 +-
python/pyspark/pandas/series.py | 32 ++++-----
python/pyspark/pandas/spark/functions.py | 31 ++++++++-
python/pyspark/pandas/tests/test_series.py | 4 ++
.../pyspark/pandas/tests/test_spark_functions.py | 54 ++++++++++++++++
python/pyspark/pandas/utils.py | 9 +--
python/pyspark/pandas/window.py | 13 ++--
25 files changed, 290 insertions(+), 171 deletions(-)
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index b8435a3..5d28b9e 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -649,6 +649,7 @@ pyspark_pandas = Module(
"pyspark.pandas.tests.test_series_conversion",
"pyspark.pandas.tests.test_series_datetime",
"pyspark.pandas.tests.test_series_string",
+ "pyspark.pandas.tests.test_spark_functions",
"pyspark.pandas.tests.test_sql",
"pyspark.pandas.tests.test_typedef",
"pyspark.pandas.tests.test_utils",
diff --git a/python/pyspark/pandas/base.py b/python/pyspark/pandas/base.py
index 65a4a1a..a1beb2f 100644
--- a/python/pyspark/pandas/base.py
+++ b/python/pyspark/pandas/base.py
@@ -42,6 +42,7 @@ from pyspark.pandas.internal import (
NATURAL_ORDER_COLUMN_NAME,
SPARK_DEFAULT_INDEX_NAME,
)
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.spark.accessors import SparkIndexOpsMethods
from pyspark.pandas.typedef import (
Dtype,
@@ -722,7 +723,7 @@ class IndexOpsMixin(object, metaclass=ABCMeta):
.agg(
F.min(F.col("__origin")).alias("__partition_min"),
F.max(F.col("__origin")).alias("__partition_max"),
- F.min(F.coalesce(F.col("__comparison_within_partition"), F.lit(True))).alias(
+ F.min(F.coalesce(F.col("__comparison_within_partition"), SF.lit(True))).alias(
"__comparison_within_partition"
),
)
@@ -747,8 +748,8 @@ class IndexOpsMixin(object, metaclass=ABCMeta):
)
ret = sdf.select(
- F.min(F.coalesce(F.col("__comparison_between_partitions"), F.lit(True)))
- & F.min(F.coalesce(F.col("__comparison_within_partition"), F.lit(True)))
+ F.min(F.coalesce(F.col("__comparison_between_partitions"), SF.lit(True)))
+ & F.min(F.coalesce(F.col("__comparison_within_partition"), SF.lit(True)))
).collect()[0][0]
if ret is None:
return True
@@ -875,7 +876,7 @@ class IndexOpsMixin(object, metaclass=ABCMeta):
)
values = values.tolist() if isinstance(values, np.ndarray) else list(values)
- return self._with_new_scol(self.spark.column.isin(values))
+ return self._with_new_scol(self.spark.column.isin([SF.lit(v) for v in values]))
def isnull(self: T_IndexOps) -> T_IndexOps:
"""
@@ -1012,7 +1013,7 @@ class IndexOpsMixin(object, metaclass=ABCMeta):
# any and every was added as of Spark 3.0
# ret = sdf.select(F.expr("every(CAST(`%s` AS BOOLEAN))" % sdf.columns[0])).collect()[0][0]
# Here we use min as its alternative:
- ret = sdf.select(F.min(F.coalesce(col.cast("boolean"), F.lit(True)))).collect()[0][0]
+ ret = sdf.select(F.min(F.coalesce(col.cast("boolean"), SF.lit(True)))).collect()[0][0]
if ret is None:
return True
else:
@@ -1075,7 +1076,7 @@ class IndexOpsMixin(object, metaclass=ABCMeta):
# any and every was added as of Spark 3.0
# ret = sdf.select(F.expr("any(CAST(`%s` AS BOOLEAN))" % sdf.columns[0])).collect()[0][0]
# Here we use max as its alternative:
- ret = sdf.select(F.max(F.coalesce(col.cast("boolean"), F.lit(False)))).collect()[0][0]
+ ret = sdf.select(F.max(F.coalesce(col.cast("boolean"), SF.lit(False)))).collect()[0][0]
if ret is None:
return False
else:
@@ -1327,7 +1328,7 @@ class IndexOpsMixin(object, metaclass=ABCMeta):
if normalize:
sum = sdf_dropna.count()
- sdf = sdf.withColumn("count", F.col("count") / F.lit(sum))
+ sdf = sdf.withColumn("count", F.col("count") / SF.lit(sum))
internal = InternalFrame(
spark_frame=sdf,
@@ -1565,12 +1566,12 @@ class IndexOpsMixin(object, metaclass=ABCMeta):
if isinstance(self.dtype, CategoricalDtype):
categories = self.dtype.categories
if len(categories) == 0:
- scol = F.lit(None)
+ scol = SF.lit(None)
else:
kvs = list(
chain(
*[
- (F.lit(code), F.lit(category))
+ (SF.lit(code), SF.lit(category))
for code, category in enumerate(categories)
]
)
@@ -1618,11 +1619,11 @@ class IndexOpsMixin(object, metaclass=ABCMeta):
code += 1
kvs = list(
- chain(*([(F.lit(unique), F.lit(code)) for unique, code in unique_to_code.items()]))
+ chain(*([(SF.lit(unique), SF.lit(code)) for unique, code in unique_to_code.items()]))
)
if len(kvs) == 0: # uniques are all missing values
- new_scol = F.lit(na_sentinel_code)
+ new_scol = SF.lit(na_sentinel_code)
else:
scol = self.spark.column
if isinstance(self.spark.data_type, (FloatType, DoubleType)):
@@ -1631,7 +1632,7 @@ class IndexOpsMixin(object, metaclass=ABCMeta):
cond = scol.isNull()
map_scol = F.create_map(*kvs)
- null_scol = F.when(cond, F.lit(na_sentinel_code))
+ null_scol = F.when(cond, SF.lit(na_sentinel_code))
new_scol = null_scol.otherwise(map_scol.getItem(scol))
codes = self._with_new_scol(new_scol.alias(self._internal.data_spark_column_names[0]))
diff --git a/python/pyspark/pandas/data_type_ops/base.py b/python/pyspark/pandas/data_type_ops/base.py
index 479f09b..0251503 100644
--- a/python/pyspark/pandas/data_type_ops/base.py
+++ b/python/pyspark/pandas/data_type_ops/base.py
@@ -42,6 +42,7 @@ from pyspark.sql.types import (
TimestampType,
UserDefinedType,
)
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.typedef import Dtype, extension_dtypes
from pyspark.pandas.typedef.typehints import (
extension_dtypes_available,
@@ -120,14 +121,14 @@ def _as_categorical_type(
else:
categories = dtype.categories
if len(categories) == 0:
- scol = F.lit(-1)
+ scol = SF.lit(-1)
else:
kvs = chain(
- *[(F.lit(category), F.lit(code)) for code, category in enumerate(categories)]
+ *[(SF.lit(category), SF.lit(code)) for code, category in enumerate(categories)]
)
map_scol = F.create_map(*kvs)
- scol = F.coalesce(map_scol.getItem(index_ops.spark.column), F.lit(-1))
+ scol = F.coalesce(map_scol.getItem(index_ops.spark.column), SF.lit(-1))
return index_ops._with_new_scol(
scol.cast(spark_type).alias(index_ops._internal.data_fields[0].name),
field=index_ops._internal.data_fields[0].copy(
@@ -143,7 +144,7 @@ def _as_bool_type(index_ops: T_IndexOps, dtype: Union[str, type, Dtype]) -> T_In
if isinstance(dtype, extension_dtypes):
scol = index_ops.spark.column.cast(BooleanType())
else:
- scol = F.when(index_ops.spark.column.isNull(), F.lit(False)).otherwise(
+ scol = F.when(index_ops.spark.column.isNull(), SF.lit(False)).otherwise(
index_ops.spark.column.cast(BooleanType())
)
return index_ops._with_new_scol(
diff --git a/python/pyspark/pandas/data_type_ops/binary_ops.py b/python/pyspark/pandas/data_type_ops/binary_ops.py
index a5ed99c..0b8875a 100644
--- a/python/pyspark/pandas/data_type_ops/binary_ops.py
+++ b/python/pyspark/pandas/data_type_ops/binary_ops.py
@@ -29,6 +29,7 @@ from pyspark.pandas.data_type_ops.base import (
_as_other_type,
_as_string_type,
)
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.typedef import Dtype, pandas_on_spark_type
from pyspark.sql import functions as F
from pyspark.sql.types import BinaryType, BooleanType, StringType
@@ -47,7 +48,7 @@ class BinaryOps(DataTypeOps):
if isinstance(right, IndexOpsMixin) and isinstance(right.spark.data_type, BinaryType):
return column_op(F.concat)(left, right)
elif isinstance(right, bytes):
- return column_op(F.concat)(left, F.lit(right))
+ return column_op(F.concat)(left, SF.lit(right))
else:
raise TypeError(
"Concatenation can not be applied to %s and the given type." % self.pretty_name
@@ -56,7 +57,7 @@ class BinaryOps(DataTypeOps):
def radd(self, left: T_IndexOps, right: Any) -> IndexOpsLike:
if isinstance(right, bytes):
return cast(
- IndexOpsLike, left._with_new_scol(F.concat(F.lit(right), left.spark.column))
+ IndexOpsLike, left._with_new_scol(F.concat(SF.lit(right), left.spark.column))
)
else:
raise TypeError(
diff --git a/python/pyspark/pandas/data_type_ops/boolean_ops.py b/python/pyspark/pandas/data_type_ops/boolean_ops.py
index a41abc1..dfac5ee 100644
--- a/python/pyspark/pandas/data_type_ops/boolean_ops.py
+++ b/python/pyspark/pandas/data_type_ops/boolean_ops.py
@@ -33,6 +33,7 @@ from pyspark.pandas.data_type_ops.base import (
_as_other_type,
)
from pyspark.pandas.internal import InternalField
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.typedef import Dtype, extension_dtypes, pandas_on_spark_type
from pyspark.pandas.typedef.typehints import as_spark_type
from pyspark.sql import functions as F
@@ -226,9 +227,9 @@ class BooleanOps(DataTypeOps):
def and_func(left: Column, right: Any) -> Column:
if not isinstance(right, Column):
if pd.isna(right):
- right = F.lit(None)
+ right = SF.lit(None)
else:
- right = F.lit(right)
+ right = SF.lit(right)
scol = left & right
return F.when(scol.isNull(), False).otherwise(scol)
@@ -241,9 +242,9 @@ class BooleanOps(DataTypeOps):
def or_func(left: Column, right: Any) -> Column:
if not isinstance(right, Column) and pd.isna(right):
- return F.lit(False)
+ return SF.lit(False)
else:
- scol = left | F.lit(right)
+ scol = left | SF.lit(right)
return F.when(left.isNull() | scol.isNull(), False).otherwise(scol)
return column_op(or_func)(left, right)
@@ -283,9 +284,9 @@ class BooleanExtensionOps(BooleanOps):
def and_func(left: Column, right: Any) -> Column:
if not isinstance(right, Column):
if pd.isna(right):
- right = F.lit(None)
+ right = SF.lit(None)
else:
- right = F.lit(right)
+ right = SF.lit(right)
return left & right
return column_op(and_func)(left, right)
@@ -294,9 +295,9 @@ class BooleanExtensionOps(BooleanOps):
def or_func(left: Column, right: Any) -> Column:
if not isinstance(right, Column):
if pd.isna(right):
- right = F.lit(None)
+ right = SF.lit(None)
else:
- right = F.lit(right)
+ right = SF.lit(right)
return left | right
return column_op(or_func)(left, right)
diff --git a/python/pyspark/pandas/data_type_ops/categorical_ops.py b/python/pyspark/pandas/data_type_ops/categorical_ops.py
index 339603a..0937816 100644
--- a/python/pyspark/pandas/data_type_ops/categorical_ops.py
+++ b/python/pyspark/pandas/data_type_ops/categorical_ops.py
@@ -22,6 +22,7 @@ import pandas as pd
from pandas.api.types import CategoricalDtype
from pyspark.pandas.data_type_ops.base import DataTypeOps, T_IndexOps
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.typedef import Dtype, pandas_on_spark_type
from pyspark.sql import functions as F
@@ -53,10 +54,10 @@ class CategoricalOps(DataTypeOps):
categories = index_ops.dtype.categories
if len(categories) == 0:
- scol = F.lit(None)
+ scol = SF.lit(None)
else:
kvs = chain(
- *[(F.lit(code), F.lit(category)) for code, category in enumerate(categories)]
+ *[(SF.lit(code), SF.lit(category)) for code, category in enumerate(categories)]
)
map_scol = F.create_map(*kvs)
scol = map_scol.getItem(index_ops.spark.column)
diff --git a/python/pyspark/pandas/data_type_ops/date_ops.py b/python/pyspark/pandas/data_type_ops/date_ops.py
index 33d5459..26c7285 100644
--- a/python/pyspark/pandas/data_type_ops/date_ops.py
+++ b/python/pyspark/pandas/data_type_ops/date_ops.py
@@ -35,6 +35,7 @@ from pyspark.pandas.data_type_ops.base import (
_as_other_type,
_as_string_type,
)
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.typedef import Dtype, pandas_on_spark_type
@@ -60,7 +61,7 @@ class DateOps(DataTypeOps):
return column_op(F.datediff)(left, right).astype("long")
elif isinstance(right, datetime.date) and not isinstance(right, datetime.datetime):
warnings.warn(msg, UserWarning)
- return column_op(F.datediff)(left, F.lit(right)).astype("long")
+ return column_op(F.datediff)(left, SF.lit(right)).astype("long")
else:
raise TypeError("date subtraction can only be applied to date series.")
@@ -74,7 +75,7 @@ class DateOps(DataTypeOps):
)
if isinstance(right, datetime.date) and not isinstance(right, datetime.datetime):
warnings.warn(msg, UserWarning)
- return -column_op(F.datediff)(left, F.lit(right)).astype("long")
+ return -column_op(F.datediff)(left, SF.lit(right)).astype("long")
else:
raise TypeError("date subtraction can only be applied to date series.")
diff --git a/python/pyspark/pandas/data_type_ops/datetime_ops.py b/python/pyspark/pandas/data_type_ops/datetime_ops.py
index 55d8fcc..508995f 100644
--- a/python/pyspark/pandas/data_type_ops/datetime_ops.py
+++ b/python/pyspark/pandas/data_type_ops/datetime_ops.py
@@ -35,6 +35,7 @@ from pyspark.pandas.data_type_ops.base import (
_as_other_type,
)
from pyspark.pandas.internal import InternalField
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.typedef import as_spark_type, Dtype, extension_dtypes, pandas_on_spark_type
@@ -63,7 +64,7 @@ class DatetimeOps(DataTypeOps):
return cast(
IndexOpsLike,
left.spark.transform(
- lambda scol: scol.astype("long") - F.lit(right).cast(as_spark_type("long"))
+ lambda scol: scol.astype("long") - SF.lit(right).cast(as_spark_type("long"))
),
)
else:
@@ -82,7 +83,7 @@ class DatetimeOps(DataTypeOps):
return cast(
IndexOpsLike,
left.spark.transform(
- lambda scol: F.lit(right).cast(as_spark_type("long")) - scol.astype("long")
+ lambda scol: SF.lit(right).cast(as_spark_type("long")) - scol.astype("long")
),
)
else:
diff --git a/python/pyspark/pandas/data_type_ops/num_ops.py b/python/pyspark/pandas/data_type_ops/num_ops.py
index d64d987..0934dd02 100644
--- a/python/pyspark/pandas/data_type_ops/num_ops.py
+++ b/python/pyspark/pandas/data_type_ops/num_ops.py
@@ -142,7 +142,7 @@ class NumericOps(DataTypeOps):
raise TypeError("exponentiation can not be applied to given types.")
def rpow_func(left: Column, right: Any) -> Column:
- return F.when(F.lit(right == 1), right).otherwise(Column.__rpow__(left, right))
+ return F.when(SF.lit(right == 1), right).otherwise(Column.__rpow__(left, right))
right = transform_boolean_operand_to_numeric(right)
return column_op(rpow_func)(left, right)
@@ -199,9 +199,9 @@ class IntegralOps(NumericOps):
right = transform_boolean_operand_to_numeric(right, left.spark.data_type)
def truediv(left: Column, right: Any) -> Column:
- return F.when(F.lit(right != 0) | F.lit(right).isNull(), left.__div__(right)).otherwise(
- F.lit(np.inf).__div__(left)
- )
+ return F.when(
+ SF.lit(right != 0) | SF.lit(right).isNull(), left.__div__(right)
+ ).otherwise(SF.lit(np.inf).__div__(left))
return numpy_column_op(truediv)(left, right)
@@ -217,10 +217,10 @@ class IntegralOps(NumericOps):
right = transform_boolean_operand_to_numeric(right, left.spark.data_type)
def floordiv(left: Column, right: Any) -> Column:
- return F.when(F.lit(right is np.nan), np.nan).otherwise(
+ return F.when(SF.lit(right is np.nan), np.nan).otherwise(
F.when(
- F.lit(right != 0) | F.lit(right).isNull(), F.floor(left.__div__(right))
- ).otherwise(F.lit(np.inf).__div__(left))
+ SF.lit(right != 0) | SF.lit(right).isNull(), F.floor(left.__div__(right))
+ ).otherwise(SF.lit(np.inf).__div__(left))
)
return numpy_column_op(floordiv)(left, right)
@@ -232,8 +232,8 @@ class IntegralOps(NumericOps):
raise TypeError("division can not be applied to given types.")
def rtruediv(left: Column, right: Any) -> Column:
- return F.when(left == 0, F.lit(np.inf).__div__(right)).otherwise(
- F.lit(right).__truediv__(left)
+ return F.when(left == 0, SF.lit(np.inf).__div__(right)).otherwise(
+ SF.lit(right).__truediv__(left)
)
right = transform_boolean_operand_to_numeric(right, left.spark.data_type)
@@ -246,8 +246,8 @@ class IntegralOps(NumericOps):
raise TypeError("division can not be applied to given types.")
def rfloordiv(left: Column, right: Any) -> Column:
- return F.when(F.lit(left == 0), F.lit(np.inf).__div__(right)).otherwise(
- F.floor(F.lit(right).__div__(left))
+ return F.when(SF.lit(left == 0), SF.lit(np.inf).__div__(right)).otherwise(
+ F.floor(SF.lit(right).__div__(left))
)
right = transform_boolean_operand_to_numeric(right, left.spark.data_type)
@@ -302,9 +302,11 @@ class FractionalOps(NumericOps):
right = transform_boolean_operand_to_numeric(right, left.spark.data_type)
def truediv(left: Column, right: Any) -> Column:
- return F.when(F.lit(right != 0) | F.lit(right).isNull(), left.__div__(right)).otherwise(
- F.when(F.lit(left == np.inf) | F.lit(left == -np.inf), left).otherwise(
- F.lit(np.inf).__div__(left)
+ return F.when(
+ SF.lit(right != 0) | SF.lit(right).isNull(), left.__div__(right)
+ ).otherwise(
+ F.when(SF.lit(left == np.inf) | SF.lit(left == -np.inf), left).otherwise(
+ SF.lit(np.inf).__div__(left)
)
)
@@ -322,12 +324,12 @@ class FractionalOps(NumericOps):
right = transform_boolean_operand_to_numeric(right, left.spark.data_type)
def floordiv(left: Column, right: Any) -> Column:
- return F.when(F.lit(right is np.nan), np.nan).otherwise(
+ return F.when(SF.lit(right is np.nan), np.nan).otherwise(
F.when(
- F.lit(right != 0) | F.lit(right).isNull(), F.floor(left.__div__(right))
+ SF.lit(right != 0) | SF.lit(right).isNull(), F.floor(left.__div__(right))
).otherwise(
- F.when(F.lit(left == np.inf) | F.lit(left == -np.inf), left).otherwise(
- F.lit(np.inf).__div__(left)
+ F.when(SF.lit(left == np.inf) | SF.lit(left == -np.inf), left).otherwise(
+ SF.lit(np.inf).__div__(left)
)
)
)
@@ -341,8 +343,8 @@ class FractionalOps(NumericOps):
raise TypeError("division can not be applied to given types.")
def rtruediv(left: Column, right: Any) -> Column:
- return F.when(left == 0, F.lit(np.inf).__div__(right)).otherwise(
- F.lit(right).__truediv__(left)
+ return F.when(left == 0, SF.lit(np.inf).__div__(right)).otherwise(
+ SF.lit(right).__truediv__(left)
)
right = transform_boolean_operand_to_numeric(right, left.spark.data_type)
@@ -355,8 +357,10 @@ class FractionalOps(NumericOps):
raise TypeError("division can not be applied to given types.")
def rfloordiv(left: Column, right: Any) -> Column:
- return F.when(F.lit(left == 0), F.lit(np.inf).__div__(right)).otherwise(
- F.when(F.lit(left) == np.nan, np.nan).otherwise(F.floor(F.lit(right).__div__(left)))
+ return F.when(SF.lit(left == 0), SF.lit(np.inf).__div__(right)).otherwise(
+ F.when(SF.lit(left) == np.nan, np.nan).otherwise(
+ F.floor(SF.lit(right).__div__(left))
+ )
)
right = transform_boolean_operand_to_numeric(right, left.spark.data_type)
@@ -381,7 +385,7 @@ class FractionalOps(NumericOps):
else:
scol = F.when(
index_ops.spark.column.isNull() | F.isnan(index_ops.spark.column),
- F.lit(True),
+ SF.lit(True),
).otherwise(index_ops.spark.column.cast(spark_type))
return index_ops._with_new_scol(
scol.alias(index_ops._internal.data_spark_column_names[0]),
diff --git a/python/pyspark/pandas/data_type_ops/string_ops.py b/python/pyspark/pandas/data_type_ops/string_ops.py
index 543ae41..cdf4c1f 100644
--- a/python/pyspark/pandas/data_type_ops/string_ops.py
+++ b/python/pyspark/pandas/data_type_ops/string_ops.py
@@ -51,7 +51,7 @@ class StringOps(DataTypeOps):
if isinstance(right, IndexOpsMixin) and isinstance(right.spark.data_type, StringType):
return column_op(F.concat)(left, right)
elif isinstance(right, str):
- return column_op(F.concat)(left, F.lit(right))
+ return column_op(F.concat)(left, SF.lit(right))
else:
raise TypeError("string addition can only be applied to string series or literals.")
@@ -87,7 +87,7 @@ class StringOps(DataTypeOps):
if isinstance(right, str):
return cast(
IndexOpsLike,
- left._with_new_scol(F.concat(F.lit(right), left.spark.column)), # TODO: dtype?
+ left._with_new_scol(F.concat(SF.lit(right), left.spark.column)), # TODO: dtype?
)
else:
raise TypeError("string addition can only be applied to string series or literals.")
@@ -123,7 +123,7 @@ class StringOps(DataTypeOps):
if isinstance(dtype, extension_dtypes):
scol = index_ops.spark.column.cast(spark_type)
else:
- scol = F.when(index_ops.spark.column.isNull(), F.lit(False)).otherwise(
+ scol = F.when(index_ops.spark.column.isNull(), SF.lit(False)).otherwise(
F.length(index_ops.spark.column) > 0
)
return index_ops._with_new_scol(
diff --git a/python/pyspark/pandas/frame.py b/python/pyspark/pandas/frame.py
index 6b6301a..8a4b364 100644
--- a/python/pyspark/pandas/frame.py
+++ b/python/pyspark/pandas/frame.py
@@ -86,6 +86,7 @@ from pyspark.sql.window import Window
from pyspark import pandas as ps # For running doctests and reference resolution in PyCharm.
from pyspark.pandas.accessors import PandasOnSparkFrameMethods
from pyspark.pandas.config import option_context, get_option
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.spark.accessors import SparkFrameMethods, CachedSparkFrameMethods
from pyspark.pandas.utils import (
align_diff_frames,
@@ -672,7 +673,7 @@ class DataFrame(Frame, Generic[T]):
if axis == 0:
min_count = kwargs.get("min_count", 0)
- exprs = [F.lit(None).cast(StringType()).alias(SPARK_DEFAULT_INDEX_NAME)]
+ exprs = [SF.lit(None).cast(StringType()).alias(SPARK_DEFAULT_INDEX_NAME)]
new_column_labels = []
num_args = len(signature(sfun).parameters)
for label in self._internal.column_labels:
@@ -829,7 +830,7 @@ class DataFrame(Frame, Generic[T]):
applied.append(getattr(self._psser_for(label), op)(other._psser_for(label)))
else:
applied.append(
- F.lit(None)
+ SF.lit(None)
.cast(self._internal.spark_type_for(label))
.alias(name_like_string(label))
)
@@ -837,7 +838,7 @@ class DataFrame(Frame, Generic[T]):
for label in other._internal.column_labels:
if label not in column_labels:
applied.append(
- F.lit(None)
+ SF.lit(None)
.cast(other._internal.spark_type_for(label))
.alias(name_like_string(label))
)
@@ -2285,7 +2286,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
*[
F.struct(
*[
- F.lit(col).alias(SPARK_INDEX_NAME_FORMAT(i))
+ SF.lit(col).alias(SPARK_INDEX_NAME_FORMAT(i))
for i, col in enumerate(label)
],
*[self._internal.spark_column_for(label).alias("value")],
@@ -3323,7 +3324,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
(
cond._internal.spark_column_for(label)
if label in cond._internal.column_labels
- else F.lit(False)
+ else SF.lit(False)
).alias(name)
for label, name in zip(self._internal.column_labels, tmp_cond_col_names)
]
@@ -3347,7 +3348,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
(
other._internal.spark_column_for(label)
if label in other._internal.column_labels
- else F.lit(np.nan)
+ else SF.lit(np.nan)
).alias(name)
for label, name in zip(self._internal.column_labels, tmp_other_col_names)
]
@@ -4250,7 +4251,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
if axis != 0:
raise NotImplementedError('axis should be either 0 or "index" currently.')
sdf = self._internal.spark_frame.select(
- [F.lit(None).cast(StringType()).alias(SPARK_DEFAULT_INDEX_NAME)]
+ [SF.lit(None).cast(StringType()).alias(SPARK_DEFAULT_INDEX_NAME)]
+ [
self._psser_for(label)._nunique(dropna, approx, rsd)
for label in self._internal.column_labels
@@ -4927,7 +4928,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
if isinstance(v, IndexOpsMixin) and not isinstance(v, MultiIndex)
else (v, None)
if isinstance(v, Column)
- else (F.lit(v), None)
+ else (SF.lit(v), None)
)
for k, v in kwargs.items()
}
@@ -5281,14 +5282,14 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
F.when(self._psser_for(label).notna().spark.column, 1).otherwise(0)
for label in labels
],
- F.lit(0),
+ SF.lit(0),
)
if thresh is not None:
- pred = cnt >= F.lit(int(thresh))
+ pred = cnt >= SF.lit(int(thresh))
elif how == "any":
- pred = cnt == F.lit(len(labels))
+ pred = cnt == SF.lit(len(labels))
elif how == "all":
- pred = cnt > F.lit(0)
+ pred = cnt > SF.lit(0)
internal = self._internal.with_filter(pred)
if inplace:
@@ -5313,7 +5314,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
reduce(
lambda x, y: x & y,
[
- scol == F.lit(l)
+ scol == SF.lit(l)
for l, scol in zip(lbl, internal.index_spark_columns)
],
)
@@ -5719,7 +5720,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
if n < 0:
n = len(self) + n
if n <= 0:
- return DataFrame(self._internal.with_filter(F.lit(False)))
+ return DataFrame(self._internal.with_filter(SF.lit(False)))
else:
sdf = self._internal.resolved_copy.spark_frame
if get_option("compute.ordered_head"):
@@ -6107,7 +6108,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
index_map = OrderedDict() # type: Dict[str, Optional[Tuple]]
for i, index_value in enumerate(index_values):
colname = SPARK_INDEX_NAME_FORMAT(i)
- sdf = sdf.withColumn(colname, F.lit(index_value))
+ sdf = sdf.withColumn(colname, SF.lit(index_value))
index_map[colname] = None
internal = InternalFrame(
spark_frame=sdf,
@@ -7416,7 +7417,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
)
else:
data_spark_columns.append(
- F.lit(False).alias(self._internal.data_spark_column_names[i])
+ SF.lit(False).alias(self._internal.data_spark_column_names[i])
)
elif is_list_like(values):
values = values.tolist() if isinstance(values, np.ndarray) else list(values)
@@ -8496,7 +8497,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
spark_type = self._internal.spark_type_for(label)
if isinstance(spark_type, DoubleType) or isinstance(spark_type, FloatType):
exprs.append(
- F.nanvl(scol, F.lit(None)).alias(self._internal.spark_column_name_for(label))
+ F.nanvl(scol, SF.lit(None)).alias(self._internal.spark_column_name_for(label))
)
column_labels.append(label)
elif isinstance(spark_type, NumericType):
@@ -8841,7 +8842,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
frame = frame.select(index_scols + scols)
temp_fill_value = verify_temp_column_name(frame, "__fill_value__")
- labels = labels.withColumn(temp_fill_value, F.lit(fill_value))
+ labels = labels.withColumn(temp_fill_value, SF.lit(fill_value))
frame_index_scols = [scol_for(frame, col) for col in frame_index_columns]
labels_index_scols = [scol_for(labels, col) for col in index_columns]
@@ -8911,7 +8912,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
if label in self._internal.column_labels:
scols_or_pssers.append(self._psser_for(label))
else:
- scols_or_pssers.append(F.lit(fill_value).alias(name_like_string(label)))
+ scols_or_pssers.append(SF.lit(fill_value).alias(name_like_string(label)))
labels.append(label)
if isinstance(columns, pd.Index):
@@ -9188,7 +9189,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
F.array(
*[
F.struct(
- *[F.lit(c).alias(name) for c, name in zip(label, var_name)],
+ *[SF.lit(c).alias(name) for c, name in zip(label, var_name)],
*[self._internal.spark_column_for(label).alias(value_name)],
)
for label in column_labels
@@ -9331,7 +9332,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
return DataFrame(
self._internal.copy(
column_label_names=self._internal.column_label_names[:-1]
- ).with_filter(F.lit(False))
+ ).with_filter(SF.lit(False))
)
column_labels = defaultdict(dict) # type: Union[defaultdict, OrderedDict]
@@ -9361,12 +9362,12 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
structs = [
F.struct(
- *[F.lit(value).alias(index_column)],
+ *[SF.lit(value).alias(index_column)],
*[
(
column_labels[label][value]
if value in column_labels[label]
- else F.lit(None)
+ else SF.lit(None)
).alias(name)
for label, name in zip(column_labels, data_columns)
],
@@ -9520,7 +9521,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
F.array(
*[
F.struct(
- *[F.lit(c).alias(name) for c, name in zip(idx, new_index_columns)],
+ *[SF.lit(c).alias(name) for c, name in zip(idx, new_index_columns)],
*[self._internal.spark_column_for(idx).alias(ser_name)],
)
for idx in column_labels
@@ -9613,7 +9614,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
column_labels = self._internal.column_labels
for label in column_labels:
scol = self._internal.spark_column_for(label)
- all_col = F.min(F.coalesce(scol.cast("boolean"), F.lit(True)))
+ all_col = F.min(F.coalesce(scol.cast("boolean"), SF.lit(True)))
applied.append(F.when(all_col.isNull(), True).otherwise(all_col))
# TODO: there is a similar logic to transpose in, for instance,
@@ -9623,7 +9624,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
for label, applied_col in zip(column_labels, applied):
cols.append(
F.struct(
- *[F.lit(col).alias(SPARK_INDEX_NAME_FORMAT(i)) for i, col in enumerate(label)],
+ *[SF.lit(col).alias(SPARK_INDEX_NAME_FORMAT(i)) for i, col in enumerate(label)],
*[applied_col.alias(value_column)],
)
)
@@ -9700,7 +9701,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
column_labels = self._internal.column_labels
for label in column_labels:
scol = self._internal.spark_column_for(label)
- all_col = F.max(F.coalesce(scol.cast("boolean"), F.lit(False)))
+ all_col = F.max(F.coalesce(scol.cast("boolean"), SF.lit(False)))
applied.append(F.when(all_col.isNull(), False).otherwise(all_col))
# TODO: there is a similar logic to transpose in, for instance,
@@ -9710,7 +9711,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
for label, applied_col in zip(column_labels, applied):
cols.append(
F.struct(
- *[F.lit(col).alias(SPARK_INDEX_NAME_FORMAT(i)) for i, col in enumerate(label)],
+ *[SF.lit(col).alias(SPARK_INDEX_NAME_FORMAT(i)) for i, col in enumerate(label)],
*[applied_col.alias(value_column)],
)
)
@@ -9908,9 +9909,9 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
col = None
for item in items:
if col is None:
- col = index_scols[0] == F.lit(item)
+ col = index_scols[0] == SF.lit(item)
else:
- col = col | (index_scols[0] == F.lit(item))
+ col = col | (index_scols[0] == SF.lit(item))
elif len(index_scols) > 1:
# for multi-index
col = None
@@ -9922,9 +9923,9 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
midx_col = None
for i, element in enumerate(item):
if midx_col is None:
- midx_col = index_scols[i] == F.lit(element)
+ midx_col = index_scols[i] == SF.lit(element)
else:
- midx_col = midx_col & (index_scols[i] == F.lit(element))
+ midx_col = midx_col & (index_scols[i] == SF.lit(element))
if col is None:
col = midx_col
else:
@@ -10860,7 +10861,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
internal_index_column = SPARK_DEFAULT_INDEX_NAME
cols = []
for i, col in enumerate(zip(*cols_dict.values())):
- cols.append(F.struct(F.lit(qq[i]).alias(internal_index_column), *col))
+ cols.append(F.struct(SF.lit(qq[i]).alias(internal_index_column), *col))
sdf = sdf.select(F.array(*cols).alias("arrays"))
# And then, explode it and manually set the index.
@@ -11317,7 +11318,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
]
sdf = self._internal.spark_frame.select(
- *[F.lit(None).cast(StringType()).alias(SPARK_DEFAULT_INDEX_NAME)], *new_columns
+ *[SF.lit(None).cast(StringType()).alias(SPARK_DEFAULT_INDEX_NAME)], *new_columns
)
# The data is expected to be small so it's fine to transpose/use default index.
@@ -11423,7 +11424,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
if n < 0:
n = len(self) + n
if n <= 0:
- return ps.DataFrame(self._internal.with_filter(F.lit(False)))
+ return ps.DataFrame(self._internal.with_filter(SF.lit(False)))
# Should use `resolved_copy` here for the case like `(psdf + 1).tail()`
sdf = self._internal.resolved_copy.spark_frame
rows = sdf.tail(n)
@@ -11598,11 +11599,11 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
for label in column_labels:
if label not in left._internal.column_labels:
- left[label] = F.lit(None).cast(DoubleType())
+ left[label] = SF.lit(None).cast(DoubleType())
left = left[column_labels]
for label in column_labels:
if label not in right._internal.column_labels:
- right[label] = F.lit(None).cast(DoubleType())
+ right[label] = SF.lit(None).cast(DoubleType())
right = right[column_labels]
return (left.copy(), right.copy()) if copy else (left, right)
diff --git a/python/pyspark/pandas/generic.py b/python/pyspark/pandas/generic.py
index a0c3f23..d6c99c2 100644
--- a/python/pyspark/pandas/generic.py
+++ b/python/pyspark/pandas/generic.py
@@ -57,6 +57,8 @@ from pyspark import pandas as ps # For running doctests and reference resolutio
from pyspark.pandas.indexing import AtIndexer, iAtIndexer, iLocIndexer, LocIndexer
from pyspark.pandas.internal import InternalFrame
from pyspark.pandas.typedef import Dtype, Scalar, spark_type_to_pandas_dtype
+from pyspark.pandas.spark import functions as SF
+from pyspark.pandas.typedef import Scalar, spark_type_to_pandas_dtype
from pyspark.pandas.utils import (
is_name_like_tuple,
is_name_like_value,
@@ -1271,7 +1273,7 @@ class Frame(object, metaclass=ABCMeta):
spark_type_to_pandas_dtype(spark_type), spark_type.simpleString()
)
)
- return F.coalesce(F.sum(spark_column), F.lit(0))
+ return F.coalesce(F.sum(spark_column), SF.lit(0))
return self._reduce_for_stat_function(
sum, name="sum", axis=axis, numeric_only=numeric_only, min_count=min_count
@@ -1348,7 +1350,7 @@ class Frame(object, metaclass=ABCMeta):
def prod(spark_column: Column, spark_type: DataType) -> Column:
if isinstance(spark_type, BooleanType):
- scol = F.min(F.coalesce(spark_column, F.lit(True))).cast(LongType())
+ scol = F.min(F.coalesce(spark_column, SF.lit(True))).cast(LongType())
elif isinstance(spark_type, NumericType):
num_zeros = F.sum(F.when(spark_column == 0, 1).otherwise(0))
sign = F.when(
@@ -1368,7 +1370,7 @@ class Frame(object, metaclass=ABCMeta):
)
)
- return F.coalesce(scol, F.lit(1))
+ return F.coalesce(scol, SF.lit(1))
return self._reduce_for_stat_function(
prod, name="prod", axis=axis, numeric_only=numeric_only, min_count=min_count
@@ -3159,7 +3161,7 @@ class Frame(object, metaclass=ABCMeta):
# Special handle floating point types because Spark's count treats nan as a valid value,
# whereas pandas count doesn't include nan.
if isinstance(spark_type, (FloatType, DoubleType)):
- return F.count(F.nanvl(spark_column, F.lit(None)))
+ return F.count(F.nanvl(spark_column, SF.lit(None)))
else:
return F.count(spark_column)
diff --git a/python/pyspark/pandas/groupby.py b/python/pyspark/pandas/groupby.py
index 1620c8c..9d3d5a1 100644
--- a/python/pyspark/pandas/groupby.py
+++ b/python/pyspark/pandas/groupby.py
@@ -75,6 +75,7 @@ from pyspark.pandas.missing.groupby import (
MissingPandasLikeSeriesGroupBy,
)
from pyspark.pandas.series import Series, first_series
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.config import get_option
from pyspark.pandas.utils import (
align_diff_frames,
@@ -566,7 +567,7 @@ class GroupBy(Generic[T_Frame], metaclass=ABCMeta):
5 False
"""
return self._reduce_for_stat_function(
- lambda col: F.min(F.coalesce(col.cast("boolean"), F.lit(True))), only_numeric=False
+ lambda col: F.min(F.coalesce(col.cast("boolean"), SF.lit(True))), only_numeric=False
)
# TODO: skipna should be implemented.
@@ -608,7 +609,7 @@ class GroupBy(Generic[T_Frame], metaclass=ABCMeta):
5 False
"""
return self._reduce_for_stat_function(
- lambda col: F.max(F.coalesce(col.cast("boolean"), F.lit(False))), only_numeric=False
+ lambda col: F.max(F.coalesce(col.cast("boolean"), SF.lit(False))), only_numeric=False
)
# TODO: groupby multiply columns should be implemented.
@@ -798,7 +799,7 @@ class GroupBy(Generic[T_Frame], metaclass=ABCMeta):
ret = (
self._groupkeys[0]
.rename()
- .spark.transform(lambda _: F.lit(0))
+ .spark.transform(lambda _: SF.lit(0))
._cum(F.count, True, part_cols=self._groupkeys_scols, ascending=ascending)
- 1
)
@@ -2427,7 +2428,7 @@ class GroupBy(Generic[T_Frame], metaclass=ABCMeta):
)
if not is_list_like(name):
name = [name]
- cond = F.lit(True)
+ cond = SF.lit(True)
for groupkey, item in zip(groupkeys, name):
scol = groupkey.spark.column
cond = cond & (scol == item)
@@ -2548,7 +2549,7 @@ class GroupBy(Generic[T_Frame], metaclass=ABCMeta):
# Special handle floating point types because Spark's count treats nan as a valid
# value, whereas pandas count doesn't include nan.
if isinstance(spark_type, DoubleType) or isinstance(spark_type, FloatType):
- stat_exprs.append(sfun(F.nanvl(scol, F.lit(None))).alias(name))
+ stat_exprs.append(sfun(F.nanvl(scol, SF.lit(None))).alias(name))
data_columns.append(name)
column_labels.append(label)
elif isinstance(spark_type, NumericType) or not only_numeric:
diff --git a/python/pyspark/pandas/indexes/base.py b/python/pyspark/pandas/indexes/base.py
index c73ea85..68e514f 100644
--- a/python/pyspark/pandas/indexes/base.py
+++ b/python/pyspark/pandas/indexes/base.py
@@ -46,6 +46,7 @@ from pyspark.pandas.base import IndexOpsMixin
from pyspark.pandas.frame import DataFrame
from pyspark.pandas.missing.indexes import MissingPandasLikeIndex
from pyspark.pandas.series import Series, first_series
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.spark.accessors import SparkIndexMethods
from pyspark.pandas.utils import (
is_name_like_tuple,
@@ -2145,7 +2146,7 @@ class Index(IndexOpsMixin):
psdf = DataFrame(self._internal.resolved_copy) # type: DataFrame
if repeats == 0:
- return DataFrame(psdf._internal.with_filter(F.lit(False))).index
+ return DataFrame(psdf._internal.with_filter(SF.lit(False))).index
else:
return ps.concat([psdf] * repeats).index
@@ -2193,11 +2194,11 @@ class Index(IndexOpsMixin):
"""
sdf = self._internal.spark_frame
if self.is_monotonic_increasing:
- sdf = sdf.where(self.spark.column <= F.lit(label).cast(self.spark.data_type)).select(
+ sdf = sdf.where(self.spark.column <= SF.lit(label).cast(self.spark.data_type)).select(
F.max(self.spark.column)
)
elif self.is_monotonic_decreasing:
- sdf = sdf.where(self.spark.column >= F.lit(label).cast(self.spark.data_type)).select(
+ sdf = sdf.where(self.spark.column >= SF.lit(label).cast(self.spark.data_type)).select(
F.min(self.spark.column)
)
else:
diff --git a/python/pyspark/pandas/indexes/multi.py b/python/pyspark/pandas/indexes/multi.py
index 15e5edf..a9e5ac9 100644
--- a/python/pyspark/pandas/indexes/multi.py
+++ b/python/pyspark/pandas/indexes/multi.py
@@ -47,6 +47,7 @@ from pyspark.pandas.internal import (
NATURAL_ORDER_COLUMN_NAME,
SPARK_INDEX_NAME_FORMAT,
)
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.typedef import Scalar
@@ -502,8 +503,8 @@ class MultiIndex(Index):
def _is_monotonic_increasing(self) -> Series:
window = Window.orderBy(NATURAL_ORDER_COLUMN_NAME).rowsBetween(-1, -1)
- cond = F.lit(True)
- has_not_null = F.lit(True)
+ cond = SF.lit(True)
+ has_not_null = SF.lit(True)
for scol in self._internal.index_spark_columns[::-1]:
data_type = self._internal.spark_type_for(scol)
prev = F.lag(scol, 1).over(window)
@@ -549,8 +550,8 @@ class MultiIndex(Index):
def _is_monotonic_decreasing(self) -> Series:
window = Window.orderBy(NATURAL_ORDER_COLUMN_NAME).rowsBetween(-1, -1)
- cond = F.lit(True)
- has_not_null = F.lit(True)
+ cond = SF.lit(True)
+ has_not_null = SF.lit(True)
for scol in self._internal.index_spark_columns[::-1]:
data_type = self._internal.spark_type_for(scol)
prev = F.lag(scol, 1).over(window)
diff --git a/python/pyspark/pandas/indexing.py b/python/pyspark/pandas/indexing.py
index 83fa5b1..ef37733 100644
--- a/python/pyspark/pandas/indexing.py
+++ b/python/pyspark/pandas/indexing.py
@@ -39,6 +39,7 @@ from pyspark.pandas.internal import (
SPARK_DEFAULT_SERIES_NAME,
)
from pyspark.pandas.exceptions import SparkPandasIndexingError, SparkPandasNotImplementedError
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.typedef.typehints import Scalar
from pyspark.pandas.utils import (
is_name_like_tuple,
@@ -626,10 +627,11 @@ class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
cond, limit, remaining_index = self._select_rows(key)
if cond is None:
- cond = F.lit(True)
+ cond = SF.lit(True)
if limit is not None:
cond = cond & (
- self._internal.spark_frame[cast(iLocIndexer, self)._sequence_col] < F.lit(limit)
+ self._internal.spark_frame[cast(iLocIndexer, self)._sequence_col]
+ < SF.lit(limit)
)
if isinstance(value, (Series, spark.Column)):
@@ -640,7 +642,7 @@ class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
if isinstance(value, Series):
value = value.spark.column
else:
- value = F.lit(value)
+ value = SF.lit(value)
scol = (
F.when(cond, value)
.otherwise(self._internal.spark_column_for(self._psdf_or_psser._column_label))
@@ -711,10 +713,11 @@ class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
_, data_spark_columns, _, _, _ = self._select_cols(cols_sel, missing_keys=missing_keys)
if cond is None:
- cond = F.lit(True)
+ cond = SF.lit(True)
if limit is not None:
cond = cond & (
- self._internal.spark_frame[cast(iLocIndexer, self)._sequence_col] < F.lit(limit)
+ self._internal.spark_frame[cast(iLocIndexer, self)._sequence_col]
+ < SF.lit(limit)
)
if isinstance(value, (Series, spark.Column)):
@@ -725,7 +728,7 @@ class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
if isinstance(value, Series):
value = value.spark.column
else:
- value = F.lit(value)
+ value = SF.lit(value)
new_data_spark_columns = []
new_fields = []
@@ -1016,8 +1019,8 @@ class LocIndexer(LocIndexerLike):
start_and_stop = (
sdf.select(index_column.spark.column, NATURAL_ORDER_COLUMN_NAME)
.where(
- (index_column.spark.column == F.lit(start).cast(index_data_type))
- | (index_column.spark.column == F.lit(stop).cast(index_data_type))
+ (index_column.spark.column == SF.lit(start).cast(index_data_type))
+ | (index_column.spark.column == SF.lit(stop).cast(index_data_type))
)
.collect()
)
@@ -1030,9 +1033,9 @@ class LocIndexer(LocIndexerLike):
conds = [] # type: List[spark.Column]
if start is not None:
- conds.append(F.col(NATURAL_ORDER_COLUMN_NAME) >= F.lit(start).cast(LongType()))
+ conds.append(F.col(NATURAL_ORDER_COLUMN_NAME) >= SF.lit(start).cast(LongType()))
if stop is not None:
- conds.append(F.col(NATURAL_ORDER_COLUMN_NAME) <= F.lit(stop).cast(LongType()))
+ conds.append(F.col(NATURAL_ORDER_COLUMN_NAME) <= SF.lit(stop).cast(LongType()))
# if index order is not monotonic increasing or decreasing
# and specified values don't exist in index, raise KeyError
@@ -1048,20 +1051,24 @@ class LocIndexer(LocIndexerLike):
start = rows_sel.start
if inc is not False:
conds.append(
- index_column.spark.column >= F.lit(start).cast(index_data_type)
+ index_column.spark.column >= SF.lit(start).cast(index_data_type)
)
elif dec is not False:
conds.append(
- index_column.spark.column <= F.lit(start).cast(index_data_type)
+ index_column.spark.column <= SF.lit(start).cast(index_data_type)
)
else:
raise KeyError(rows_sel.start)
if stop is None and rows_sel.stop is not None:
stop = rows_sel.stop
if inc is not False:
- conds.append(index_column.spark.column <= F.lit(stop).cast(index_data_type))
+ conds.append(
+ index_column.spark.column <= SF.lit(stop).cast(index_data_type)
+ )
elif dec is not False:
- conds.append(index_column.spark.column >= F.lit(stop).cast(index_data_type))
+ conds.append(
+ index_column.spark.column >= SF.lit(stop).cast(index_data_type)
+ )
else:
raise KeyError(rows_sel.stop)
@@ -1098,23 +1105,23 @@ class LocIndexer(LocIndexerLike):
conds = []
if start is not None:
- cond = F.lit(True)
+ cond = SF.lit(True)
for scol, value, dt in list(
zip(self._internal.index_spark_columns, start, index_data_type)
)[::-1]:
compare = MultiIndex._comparator_for_monotonic_increasing(dt)
- cond = F.when(scol.eqNullSafe(F.lit(value).cast(dt)), cond).otherwise(
- compare(scol, F.lit(value).cast(dt), spark.Column.__gt__)
+ cond = F.when(scol.eqNullSafe(SF.lit(value).cast(dt)), cond).otherwise(
+ compare(scol, SF.lit(value).cast(dt), spark.Column.__gt__)
)
conds.append(cond)
if stop is not None:
- cond = F.lit(True)
+ cond = SF.lit(True)
for scol, value, dt in list(
zip(self._internal.index_spark_columns, stop, index_data_type)
)[::-1]:
compare = MultiIndex._comparator_for_monotonic_increasing(dt)
- cond = F.when(scol.eqNullSafe(F.lit(value).cast(dt)), cond).otherwise(
- compare(scol, F.lit(value).cast(dt), spark.Column.__lt__)
+ cond = F.when(scol.eqNullSafe(SF.lit(value).cast(dt)), cond).otherwise(
+ compare(scol, SF.lit(value).cast(dt), spark.Column.__lt__)
)
conds.append(cond)
@@ -1125,20 +1132,20 @@ class LocIndexer(LocIndexerLike):
) -> Tuple[Optional[spark.Column], Optional[int], Optional[int]]:
rows_sel = list(rows_sel)
if len(rows_sel) == 0:
- return F.lit(False), None, None
+ return SF.lit(False), None, None
elif self._internal.index_level == 1:
index_column = self._psdf_or_psser.index.to_series()
index_data_type = index_column.spark.data_type
if len(rows_sel) == 1:
return (
- index_column.spark.column == F.lit(rows_sel[0]).cast(index_data_type),
+ index_column.spark.column == SF.lit(rows_sel[0]).cast(index_data_type),
None,
None,
)
else:
return (
index_column.spark.column.isin(
- [F.lit(r).cast(index_data_type) for r in rows_sel]
+ [SF.lit(r).cast(index_data_type) for r in rows_sel]
),
None,
None,
@@ -1618,22 +1625,22 @@ class iLocIndexer(LocIndexerLike):
if start < 0:
start = start + cnt
if step >= 0:
- cond.append(sequence_scol >= F.lit(start).cast(LongType()))
+ cond.append(sequence_scol >= SF.lit(start).cast(LongType()))
else:
- cond.append(sequence_scol <= F.lit(start).cast(LongType()))
+ cond.append(sequence_scol <= SF.lit(start).cast(LongType()))
if stop is not None:
if stop < 0:
stop = stop + cnt
if step >= 0:
- cond.append(sequence_scol < F.lit(stop).cast(LongType()))
+ cond.append(sequence_scol < SF.lit(stop).cast(LongType()))
else:
- cond.append(sequence_scol > F.lit(stop).cast(LongType()))
+ cond.append(sequence_scol > SF.lit(stop).cast(LongType()))
if step != 1:
if step > 0:
start = start or 0
else:
start = start or (cnt - 1)
- cond.append(((sequence_scol - start) % F.lit(step).cast(LongType())) == F.lit(0))
+ cond.append(((sequence_scol - start) % SF.lit(step).cast(LongType())) == SF.lit(0))
return reduce(lambda x, y: x & y, cond), None, None
@@ -1668,10 +1675,10 @@ class iLocIndexer(LocIndexerLike):
sequence_scol = sdf[self._sequence_col]
cond = []
for key in new_rows_sel:
- cond.append(sequence_scol == F.lit(int(key)).cast(LongType()))
+ cond.append(sequence_scol == SF.lit(int(key)).cast(LongType()))
if len(cond) == 0:
- cond = [F.lit(False)]
+ cond = [SF.lit(False)]
return reduce(lambda x, y: x | y, cond), None, None
def _select_rows_else(
diff --git a/python/pyspark/pandas/namespace.py b/python/pyspark/pandas/namespace.py
index d1ecdb0..a21194f 100644
--- a/python/pyspark/pandas/namespace.py
+++ b/python/pyspark/pandas/namespace.py
@@ -83,6 +83,7 @@ from pyspark.pandas.internal import (
HIDDEN_COLUMNS,
)
from pyspark.pandas.series import Series, first_series
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.spark.utils import as_nullable_spark_type, force_decimal_precision_scale
from pyspark.pandas.typedef.typehints import Dtype
from pyspark.pandas.indexes import Index, DatetimeIndex
@@ -2376,7 +2377,7 @@ def concat(
# TODO: NaN and None difference for missing values. pandas seems filling NaN.
sdf = psdf._internal.resolved_copy.spark_frame
for label in columns_to_add:
- sdf = sdf.withColumn(name_like_string(label), F.lit(None))
+ sdf = sdf.withColumn(name_like_string(label), SF.lit(None))
data_columns = psdf._internal.data_spark_column_names + [
name_like_string(label) for label in columns_to_add
diff --git a/python/pyspark/pandas/numpy_compat.py b/python/pyspark/pandas/numpy_compat.py
index b5488dd..d487a1b 100644
--- a/python/pyspark/pandas/numpy_compat.py
+++ b/python/pyspark/pandas/numpy_compat.py
@@ -23,6 +23,7 @@ from pyspark.sql.pandas.functions import pandas_udf
from pyspark.sql.types import DoubleType, LongType, BooleanType
from pyspark.pandas.base import IndexOpsMixin
+from pyspark.pandas.spark import functions as SF
unary_np_spark_mappings = OrderedDict(
@@ -214,7 +215,7 @@ def maybe_dispatch_ufunc_to_spark_func(
@no_type_check
def convert_arguments(*args):
args = [ # type: ignore
- F.lit(inp) if not isinstance(inp, Column) else inp for inp in args
+ SF.lit(inp) if not isinstance(inp, Column) else inp for inp in args
] # type: ignore
return np_spark_map_func(*args)
diff --git a/python/pyspark/pandas/plot/core.py b/python/pyspark/pandas/plot/core.py
index 72b8ac5..839b22a 100644
--- a/python/pyspark/pandas/plot/core.py
+++ b/python/pyspark/pandas/plot/core.py
@@ -27,6 +27,7 @@ from pandas.core.dtypes.inference import is_integer
from pyspark.pandas.missing import unsupported_function
from pyspark.pandas.config import get_option
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.utils import name_like_string
@@ -184,12 +185,12 @@ class HistogramPlotBase:
if output_df is None:
output_df = bucket_df.select(
- F.lit(group_id).alias("__group_id"), F.col(bucket_name).alias("__bucket")
+ SF.lit(group_id).alias("__group_id"), F.col(bucket_name).alias("__bucket")
)
else:
output_df = output_df.union(
bucket_df.select(
- F.lit(group_id).alias("__group_id"), F.col(bucket_name).alias("__bucket")
+ SF.lit(group_id).alias("__group_id"), F.col(bucket_name).alias("__bucket")
)
)
diff --git a/python/pyspark/pandas/series.py b/python/pyspark/pandas/series.py
index b4e95ad..d7e4ccd 100644
--- a/python/pyspark/pandas/series.py
+++ b/python/pyspark/pandas/series.py
@@ -101,6 +101,7 @@ from pyspark.pandas.utils import (
SPARK_CONF_ARROW_ENABLED,
)
from pyspark.pandas.datetimes import DatetimeMethods
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.spark.accessors import SparkSeriesMethods
from pyspark.pandas.strings import StringMethods
from pyspark.pandas.typedef import (
@@ -1018,21 +1019,21 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
if isinstance(arg, dict):
is_start = True
# In case dictionary is empty.
- current = F.when(F.lit(False), F.lit(None).cast(self.spark.data_type))
+ current = F.when(SF.lit(False), SF.lit(None).cast(self.spark.data_type))
for to_replace, value in arg.items():
if is_start:
- current = F.when(self.spark.column == F.lit(to_replace), value)
+ current = F.when(self.spark.column == SF.lit(to_replace), value)
is_start = False
else:
- current = current.when(self.spark.column == F.lit(to_replace), value)
+ current = current.when(self.spark.column == SF.lit(to_replace), value)
if hasattr(arg, "__missing__"):
tmp_val = arg[np._NoValue]
del arg[np._NoValue] # Remove in case it's set in defaultdict.
- current = current.otherwise(F.lit(tmp_val))
+ current = current.otherwise(SF.lit(tmp_val))
else:
- current = current.otherwise(F.lit(None).cast(self.spark.data_type))
+ current = current.otherwise(SF.lit(None).cast(self.spark.data_type))
return self._with_new_scol(current)
else:
return self.apply(arg)
@@ -2738,7 +2739,7 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
internal = self._internal.resolved_copy
sdf = internal.spark_frame.select(
[
- F.concat(F.lit(prefix), index_spark_column).alias(index_spark_column_name)
+ F.concat(SF.lit(prefix), index_spark_column).alias(index_spark_column_name)
for index_spark_column, index_spark_column_name in zip(
internal.index_spark_columns, internal.index_spark_column_names
)
@@ -2793,7 +2794,7 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
internal = self._internal.resolved_copy
sdf = internal.spark_frame.select(
[
- F.concat(index_spark_column, F.lit(suffix)).alias(index_spark_column_name)
+ F.concat(index_spark_column, SF.lit(suffix)).alias(index_spark_column_name)
for index_spark_column, index_spark_column_name in zip(
internal.index_spark_columns, internal.index_spark_column_names
)
@@ -4061,7 +4062,7 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
return pdf[internal.data_spark_column_names[0]].iloc[0]
item_string = name_like_string(item)
- sdf = sdf.withColumn(SPARK_DEFAULT_INDEX_NAME, F.lit(str(item_string)))
+ sdf = sdf.withColumn(SPARK_DEFAULT_INDEX_NAME, SF.lit(str(item_string)))
internal = InternalFrame(
spark_frame=sdf,
index_spark_columns=[scol_for(sdf, SPARK_DEFAULT_INDEX_NAME)],
@@ -4406,7 +4407,7 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
cond = (
(F.isnan(self.spark.column) | self.spark.column.isNull())
if pd.isna(to_replace_)
- else (self.spark.column == F.lit(to_replace_))
+ else (self.spark.column == SF.lit(to_replace_))
)
if is_start:
current = F.when(cond, value)
@@ -5070,7 +5071,7 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
psdf = self._psdf[[self.name]]
if repeats == 0:
- return first_series(DataFrame(psdf._internal.with_filter(F.lit(False))))
+ return first_series(DataFrame(psdf._internal.with_filter(SF.lit(False))))
else:
return first_series(ps.concat([psdf] * repeats))
@@ -5148,7 +5149,7 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
index_scol = self._internal.index_spark_columns[0]
index_type = self._internal.spark_type_for(index_scol)
cond = [
- F.max(F.when(index_scol <= F.lit(index).cast(index_type), self.spark.column))
+ F.max(F.when(index_scol <= SF.lit(index).cast(index_type), self.spark.column))
for index in where
]
sdf = self._internal.spark_frame.select(cond)
@@ -5585,7 +5586,8 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
psser = first_series(DataFrame(internal))
return cast(
- Series, ps.concat([psser, self.loc[self.isnull()].spark.transform(lambda _: F.lit(-1))])
+ Series,
+ ps.concat([psser, self.loc[self.isnull()].spark.transform(lambda _: SF.lit(-1))]),
)
def argmax(self) -> int:
@@ -6064,7 +6066,7 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
scol = F.when(
# Manually sets nulls given the column defined above.
self.spark.column.isNull(),
- F.lit(None),
+ SF.lit(None),
).otherwise(func(self.spark.column).over(window))
else:
# Here, we use two Windows.
@@ -6100,7 +6102,7 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
# By going through with max, it sets True after the first time it meets null.
F.max(self.spark.column.isNull()).over(window),
# Manually sets nulls given the column defined above.
- F.lit(None),
+ SF.lit(None),
).otherwise(func(self.spark.column).over(window))
return self._with_new_scol(scol)
@@ -6121,7 +6123,7 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
def _cumprod(self, skipna: bool, part_cols: Sequence[Union[str, Column]] = ()) -> "Series":
if isinstance(self.spark.data_type, BooleanType):
scol = self._cum(
- lambda scol: F.min(F.coalesce(scol, F.lit(True))), skipna, part_cols
+ lambda scol: F.min(F.coalesce(scol, SF.lit(True))), skipna, part_cols
).spark.column.cast(LongType())
elif isinstance(self.spark.data_type, NumericType):
num_zeros = self._cum(
diff --git a/python/pyspark/pandas/spark/functions.py b/python/pyspark/pandas/spark/functions.py
index 27d8bd5..92ba8cc 100644
--- a/python/pyspark/pandas/spark/functions.py
+++ b/python/pyspark/pandas/spark/functions.py
@@ -17,10 +17,19 @@
"""
Additional Spark functions used in pandas-on-Spark.
"""
-from typing import Union, no_type_check
+from typing import Any, Union, no_type_check
+
+import numpy as np
from pyspark import SparkContext
+from pyspark.sql import functions as F
from pyspark.sql.column import Column, _to_java_column, _create_column_from_literal # type: ignore
+from pyspark.sql.types import (
+ ByteType,
+ FloatType,
+ IntegerType,
+ LongType,
+)
def repeat(col: Column, n: Union[int, Column]) -> Column:
@@ -32,6 +41,26 @@ def repeat(col: Column, n: Union[int, Column]) -> Column:
return _call_udf(sc, "repeat", _to_java_column(col), n)
+def lit(literal: Any) -> Column:
+ """
+ Creates a Column of literal value.
+ """
+ if isinstance(literal, np.generic):
+ scol = F.lit(literal.item())
+ if isinstance(literal, np.int64):
+ return scol.astype(LongType())
+ elif isinstance(literal, np.int32):
+ return scol.astype(IntegerType())
+ elif isinstance(literal, np.int8) or isinstance(literal, np.byte):
+ return scol.astype(ByteType())
+ elif isinstance(literal, np.float32):
+ return scol.astype(FloatType())
+ else: # TODO: Complete mappings between numpy literals and Spark data types
+ return scol
+ else:
+ return F.lit(literal)
+
+
@no_type_check
def _call_udf(sc, name, *cols):
return Column(sc._jvm.functions.callUDF(name, _make_arguments(sc, *cols)))
diff --git a/python/pyspark/pandas/tests/test_series.py b/python/pyspark/pandas/tests/test_series.py
index 5fea71d..3bb06215 100644
--- a/python/pyspark/pandas/tests/test_series.py
+++ b/python/pyspark/pandas/tests/test_series.py
@@ -433,6 +433,10 @@ class SeriesTest(PandasOnSparkTestCase, SQLTestUtils):
self.assert_eq(psser.isin(np.array(["cow", "lama"])), pser.isin(np.array(["cow", "lama"])))
self.assert_eq(psser.isin({"cow"}), pser.isin({"cow"}))
+ pser = pd.Series([np.int64(1), np.int32(1), 1])
+ psser = ps.from_pandas(pser)
+ self.assert_eq(psser.isin([np.int64(1)]), pser.isin([np.int64(1)]))
+
msg = "only list-like objects are allowed to be passed to isin()"
with self.assertRaisesRegex(TypeError, msg):
psser.isin(1)
diff --git a/python/pyspark/pandas/tests/test_spark_functions.py b/python/pyspark/pandas/tests/test_spark_functions.py
new file mode 100644
index 0000000..a8967e7
--- /dev/null
+++ b/python/pyspark/pandas/tests/test_spark_functions.py
@@ -0,0 +1,54 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import numpy as np
+
+from pyspark.pandas.spark import functions as SF
+from pyspark.pandas.utils import spark_column_equals
+from pyspark.sql import functions as F
+from pyspark.sql.types import (
+ ByteType,
+ FloatType,
+ IntegerType,
+ LongType,
+)
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
+
+
+class SparkFunctionsTests(PandasOnSparkTestCase):
+ def test_lit(self):
+ self.assertTrue(spark_column_equals(SF.lit(np.int64(1)), F.lit(1).astype(LongType())))
+ self.assertTrue(spark_column_equals(SF.lit(np.int32(1)), F.lit(1).astype(IntegerType())))
+ self.assertTrue(spark_column_equals(SF.lit(np.int8(1)), F.lit(1).astype(ByteType())))
+ self.assertTrue(spark_column_equals(SF.lit(np.byte(1)), F.lit(1).astype(ByteType())))
+ self.assertTrue(
+ spark_column_equals(SF.lit(np.float32(1)), F.lit(float(1)).astype(FloatType()))
+ )
+ self.assertTrue(spark_column_equals(SF.lit(1), F.lit(1)))
+
+
+if __name__ == "__main__":
+ import unittest
+ from pyspark.pandas.tests.test_spark_functions import * # noqa: F401
+
+ try:
+ import xmlrunner # type: ignore[import]
+
+ testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
+ except ImportError:
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/pandas/utils.py b/python/pyspark/pandas/utils.py
index 6f2ed7b..01047e9 100644
--- a/python/pyspark/pandas/utils.py
+++ b/python/pyspark/pandas/utils.py
@@ -47,6 +47,7 @@ from pandas.api.types import is_list_like
# For running doctests and reference resolution in PyCharm.
from pyspark import pandas as ps # noqa: F401
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.typedef.typehints import as_spark_type
if TYPE_CHECKING:
@@ -408,7 +409,7 @@ def align_diff_frames(
# is intentional so that `this_columns` and `that_columns` can be paired.
additional_that_columns.append(combined_label)
elif fillna:
- columns_to_keep.append(F.lit(None).cast(DoubleType()).alias(str(combined_label)))
+ columns_to_keep.append(SF.lit(None).cast(DoubleType()).alias(str(combined_label)))
column_labels_to_keep.append(combined_label)
else:
columns_to_keep.append(combined._psser_for(combined_label))
@@ -880,11 +881,11 @@ def spark_column_equals(left: spark.Column, right: spark.Column) -> bool:
"""
Check both `left` and `right` have the same expressions.
- >>> spark_column_equals(F.lit(0), F.lit(0))
+ >>> spark_column_equals(SF.lit(0), SF.lit(0))
True
- >>> spark_column_equals(F.lit(0) + 1, F.lit(0) + 1)
+ >>> spark_column_equals(SF.lit(0) + 1, SF.lit(0) + 1)
True
- >>> spark_column_equals(F.lit(0) + 1, F.lit(0) + 2)
+ >>> spark_column_equals(SF.lit(0) + 1, SF.lit(0) + 2)
False
>>> sdf1 = ps.DataFrame({"x": ['a', 'b', 'c']}).to_spark()
>>> spark_column_equals(sdf1["x"] + 1, sdf1["x"] + 1)
diff --git a/python/pyspark/pandas/window.py b/python/pyspark/pandas/window.py
index b1ee83f..9add99d 100644
--- a/python/pyspark/pandas/window.py
+++ b/python/pyspark/pandas/window.py
@@ -38,6 +38,7 @@ from pyspark.pandas.missing.window import (
from pyspark import pandas as ps # noqa: F401
from pyspark.pandas.internal import NATURAL_ORDER_COLUMN_NAME, SPARK_INDEX_NAME_FORMAT
+from pyspark.pandas.spark import functions as SF
from pyspark.pandas.utils import scol_for
from pyspark.sql.column import Column
from pyspark.sql.window import WindowSpec
@@ -76,7 +77,7 @@ class RollingAndExpanding(Generic[T_Frame], metaclass=ABCMeta):
return F.when(
F.row_number().over(self._unbounded_window) >= self._min_periods,
F.sum(scol).over(self._window),
- ).otherwise(F.lit(None))
+ ).otherwise(SF.lit(None))
return self._apply_as_series_or_frame(sum)
@@ -85,7 +86,7 @@ class RollingAndExpanding(Generic[T_Frame], metaclass=ABCMeta):
return F.when(
F.row_number().over(self._unbounded_window) >= self._min_periods,
F.min(scol).over(self._window),
- ).otherwise(F.lit(None))
+ ).otherwise(SF.lit(None))
return self._apply_as_series_or_frame(min)
@@ -94,7 +95,7 @@ class RollingAndExpanding(Generic[T_Frame], metaclass=ABCMeta):
return F.when(
F.row_number().over(self._unbounded_window) >= self._min_periods,
F.max(scol).over(self._window),
- ).otherwise(F.lit(None))
+ ).otherwise(SF.lit(None))
return self._apply_as_series_or_frame(max)
@@ -103,7 +104,7 @@ class RollingAndExpanding(Generic[T_Frame], metaclass=ABCMeta):
return F.when(
F.row_number().over(self._unbounded_window) >= self._min_periods,
F.mean(scol).over(self._window),
- ).otherwise(F.lit(None))
+ ).otherwise(SF.lit(None))
return self._apply_as_series_or_frame(mean)
@@ -112,7 +113,7 @@ class RollingAndExpanding(Generic[T_Frame], metaclass=ABCMeta):
return F.when(
F.row_number().over(self._unbounded_window) >= self._min_periods,
F.stddev(scol).over(self._window),
- ).otherwise(F.lit(None))
+ ).otherwise(SF.lit(None))
return self._apply_as_series_or_frame(std)
@@ -121,7 +122,7 @@ class RollingAndExpanding(Generic[T_Frame], metaclass=ABCMeta):
return F.when(
F.row_number().over(self._unbounded_window) >= self._min_periods,
F.variance(scol).over(self._window),
- ).otherwise(F.lit(None))
+ ).otherwise(SF.lit(None))
return self._apply_as_series_or_frame(var)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org