You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2022/11/25 01:32:57 UTC
[spark] branch master updated: [SPARK-41230][CONNECT][PYTHON] Remove `str` from Aggregate expression type
This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a205e97ad9a [SPARK-41230][CONNECT][PYTHON] Remove `str` from Aggregate expression type
a205e97ad9a is described below
commit a205e97ad9ae7894b2ec27e5253da21c4500fc8c
Author: Rui Wang <ru...@databricks.com>
AuthorDate: Fri Nov 25 09:32:41 2022 +0800
[SPARK-41230][CONNECT][PYTHON] Remove `str` from Aggregate expression type
### What changes were proposed in this pull request?
This PR proposes that Relations (e.g. Aggregate in this PR) should only deal with `Expression` than `str`. `str` could be mapped to different expressions (e.g. sql expression, unresolved_attribute, etc.). Relations are not supposed to understand the difference of `str` but DataFrame should understand it.
This PR specifically changes for `Aggregate`.
### Why are the changes needed?
Codebase refactoring.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
UT
Closes #38768 from amaliujia/SPARK-41230.
Authored-by: Rui Wang <ru...@databricks.com>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
python/pyspark/sql/connect/dataframe.py | 61 ++++++++++++----------
python/pyspark/sql/connect/plan.py | 19 ++-----
.../sql/tests/connect/test_connect_basic.py | 7 +++
3 files changed, 43 insertions(+), 44 deletions(-)
diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py
index bd374dcf814..e3a7e8c7335 100644
--- a/python/pyspark/sql/connect/dataframe.py
+++ b/python/pyspark/sql/connect/dataframe.py
@@ -37,6 +37,7 @@ from pyspark.sql.connect.column import (
Expression,
LiteralExpression,
SQLExpression,
+ ScalarFunctionExpression,
)
from pyspark.sql.types import (
StructType,
@@ -48,25 +49,13 @@ if TYPE_CHECKING:
from pyspark.sql.connect.client import RemoteSparkSession
-class GroupingFrame(object):
-
- MeasuresType = Union[Sequence[Tuple["ExpressionOrString", str]], Dict[str, str]]
- OptMeasuresType = Optional[MeasuresType]
-
+class GroupedData(object):
def __init__(self, df: "DataFrame", *grouping_cols: Union[Column, str]) -> None:
self._df = df
self._grouping_cols = [x if isinstance(x, Column) else df[x] for x in grouping_cols]
- def agg(self, exprs: Optional[MeasuresType] = None) -> "DataFrame":
-
- # Normalize the dictionary into a list of tuples.
- if isinstance(exprs, Dict):
- measures = list(exprs.items())
- elif isinstance(exprs, List):
- measures = exprs
- else:
- measures = []
-
+ def agg(self, measures: Sequence[Expression]) -> "DataFrame":
+ assert len(measures) > 0, "exprs should not be empty"
res = DataFrame.withPlan(
plan.Aggregate(
child=self._df._plan,
@@ -77,23 +66,27 @@ class GroupingFrame(object):
)
return res
- def _map_cols_to_dict(self, fun: str, cols: List[Union[Column, str]]) -> Dict[str, str]:
- return {x if isinstance(x, str) else x.name(): fun for x in cols}
+ def _map_cols_to_expression(
+ self, fun: str, col: Union[Expression, str]
+ ) -> Sequence[Expression]:
+ return [
+ ScalarFunctionExpression(fun, Column(col)) if isinstance(col, str) else col,
+ ]
- def min(self, *cols: Union[Column, str]) -> "DataFrame":
- expr = self._map_cols_to_dict("min", list(cols))
+ def min(self, col: Union[Expression, str]) -> "DataFrame":
+ expr = self._map_cols_to_expression("min", col)
return self.agg(expr)
- def max(self, *cols: Union[Column, str]) -> "DataFrame":
- expr = self._map_cols_to_dict("max", list(cols))
+ def max(self, col: Union[Expression, str]) -> "DataFrame":
+ expr = self._map_cols_to_expression("max", col)
return self.agg(expr)
- def sum(self, *cols: Union[Column, str]) -> "DataFrame":
- expr = self._map_cols_to_dict("sum", list(cols))
+ def sum(self, col: Union[Expression, str]) -> "DataFrame":
+ expr = self._map_cols_to_expression("sum", col)
return self.agg(expr)
def count(self) -> "DataFrame":
- return self.agg([(LiteralExpression(1), "count")])
+ return self.agg([ScalarFunctionExpression("count", LiteralExpression(1))])
class DataFrame(object):
@@ -162,8 +155,18 @@ class DataFrame(object):
return DataFrame.withPlan(plan.Project(self._plan, *sql_expr), session=self._session)
- def agg(self, exprs: Optional[GroupingFrame.MeasuresType]) -> "DataFrame":
- return self.groupBy().agg(exprs)
+ def agg(self, *exprs: Union[Expression, Dict[str, str]]) -> "DataFrame":
+ if not exprs:
+ raise ValueError("Argument 'exprs' must not be empty")
+
+ if len(exprs) == 1 and isinstance(exprs[0], dict):
+ measures = [ScalarFunctionExpression(f, Column(e)) for e, f in exprs[0].items()]
+ return self.groupBy().agg(measures)
+ else:
+ # other expressions
+ assert all(isinstance(c, Expression) for c in exprs), "all exprs should be Expression"
+ exprs = cast(Tuple[Expression, ...], exprs)
+ return self.groupBy().agg(exprs)
def alias(self, alias: str) -> "DataFrame":
return DataFrame.withPlan(plan.SubqueryAlias(self._plan, alias), session=self._session)
@@ -208,7 +211,7 @@ class DataFrame(object):
def count(self) -> int:
"""Returns the number of rows in the data frame"""
- pdd = self.agg([(LiteralExpression(1), "count")]).toPandas()
+ pdd = self.agg(ScalarFunctionExpression("count", LiteralExpression(1))).toPandas()
if pdd is None:
raise Exception("Empty result")
return pdd.iloc[0, 0]
@@ -340,8 +343,8 @@ class DataFrame(object):
"""
return self.head()
- def groupBy(self, *cols: "ColumnOrName") -> GroupingFrame:
- return GroupingFrame(self, *cols)
+ def groupBy(self, *cols: "ColumnOrName") -> GroupedData:
+ return GroupedData(self, *cols)
@overload
def head(self) -> Optional[Row]:
diff --git a/python/pyspark/sql/connect/plan.py b/python/pyspark/sql/connect/plan.py
index 8aadc3dc4fa..853b1a6dc0e 100644
--- a/python/pyspark/sql/connect/plan.py
+++ b/python/pyspark/sql/connect/plan.py
@@ -20,7 +20,6 @@ from typing import (
List,
Optional,
Sequence,
- Tuple,
Union,
cast,
TYPE_CHECKING,
@@ -558,29 +557,19 @@ class Sample(LogicalPlan):
class Aggregate(LogicalPlan):
- MeasureType = Tuple["ExpressionOrString", str]
- MeasuresType = Sequence[MeasureType]
- OptMeasuresType = Optional[MeasuresType]
-
def __init__(
self,
child: Optional["LogicalPlan"],
grouping_cols: List[Column],
- measures: OptMeasuresType,
+ measures: Sequence[Expression],
) -> None:
super().__init__(child)
self.grouping_cols = grouping_cols
- self.measures = measures if measures is not None else []
+ self.measures = measures
- def _convert_measure(self, m: MeasureType, session: "RemoteSparkSession") -> proto.Expression:
- exp, fun = m
+ def _convert_measure(self, m: Expression, session: "RemoteSparkSession") -> proto.Expression:
proto_expr = proto.Expression()
- measure = proto_expr.unresolved_function
- measure.parts.append(fun)
- if type(exp) is str:
- measure.arguments.append(self.unresolved_attr(exp))
- else:
- measure.arguments.append(cast(Expression, exp).to_plan(session))
+ proto_expr.CopyFrom(m.to_plan(session))
return proto_expr
def plan(self, session: "RemoteSparkSession") -> proto.Relation:
diff --git a/python/pyspark/sql/tests/connect/test_connect_basic.py b/python/pyspark/sql/tests/connect/test_connect_basic.py
index ab454c53491..529d53ed7bc 100644
--- a/python/pyspark/sql/tests/connect/test_connect_basic.py
+++ b/python/pyspark/sql/tests/connect/test_connect_basic.py
@@ -518,6 +518,13 @@ class SparkConnectTests(SparkConnectSQLTestCase):
self.connect.range(1, 10).select(col("id").alias("this", "is", "not")).collect()
self.assertIn("(this, is, not)", str(exc.exception))
+ def test_agg_with_two_agg_exprs(self):
+ # SPARK-41230: test dataframe.agg()
+ self.assert_eq(
+ self.connect.read.table(self.tbl_name).agg({"name": "min", "id": "max"}).toPandas(),
+ self.spark.read.table(self.tbl_name).agg({"name": "min", "id": "max"}).toPandas(),
+ )
+
class ChannelBuilderTests(ReusedPySparkTestCase):
def test_invalid_connection_strings(self):
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org