You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2023/03/29 00:27:06 UTC
[spark] branch master updated: [SPARK-42929][CONNECT][FOLLOWUP] Rename isBarrier to barrier
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new aacac4642ec [SPARK-42929][CONNECT][FOLLOWUP] Rename isBarrier to barrier
aacac4642ec is described below
commit aacac4642ec6462678927621809e5d30cc3b6a29
Author: Takuya UESHIN <ue...@databricks.com>
AuthorDate: Wed Mar 29 09:26:49 2023 +0900
[SPARK-42929][CONNECT][FOLLOWUP] Rename isBarrier to barrier
### What changes were proposed in this pull request?
This is a follow-up of #40559 and #40571.
Renames `isBarrier` to `barrier` in Spark Connect, too.
### Why are the changes needed?
#40571 changed the argument name from `isBarrier` to `barrier`, so Spark Connect should follow it.
### Does this PR introduce _any_ user-facing change?
Yes, it renames the parameter.
### How was this patch tested?
Existing tests.
Closes #40579 from ueshin/issues/SPARK-42929/barrier.
Authored-by: Takuya UESHIN <ue...@databricks.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../common/src/main/protobuf/spark/connect/relations.proto | 2 +-
python/pyspark/sql/connect/dataframe.py | 14 ++++++--------
python/pyspark/sql/connect/proto/relations_pb2.pyi | 2 +-
3 files changed, 8 insertions(+), 10 deletions(-)
diff --git a/connector/connect/common/src/main/protobuf/spark/connect/relations.proto b/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
index c965a6c8d32..87454dd0024 100644
--- a/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
+++ b/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
@@ -795,7 +795,7 @@ message MapPartitions {
// (Required) Input user-defined function.
CommonInlineUserDefinedFunction func = 2;
- // (Optional) isBarrier.
+ // (Optional) Whether to use barrier mode execution or not.
optional bool is_barrier = 3;
}
diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py
index 65f270f21d4..94daaa4d57d 100644
--- a/python/pyspark/sql/connect/dataframe.py
+++ b/python/pyspark/sql/connect/dataframe.py
@@ -1645,7 +1645,7 @@ class DataFrame:
func: "PandasMapIterFunction",
schema: Union[StructType, str],
evalType: int,
- is_barrier: bool,
+ barrier: bool,
) -> "DataFrame":
from pyspark.sql.connect.udf import UserDefinedFunction
@@ -1660,7 +1660,7 @@ class DataFrame:
return DataFrame.withPlan(
plan.MapPartitions(
- child=self._plan, function=udf_obj, cols=self.columns, is_barrier=is_barrier
+ child=self._plan, function=udf_obj, cols=self.columns, is_barrier=barrier
),
session=self._session,
)
@@ -1669,11 +1669,9 @@ class DataFrame:
self,
func: "PandasMapIterFunction",
schema: Union[StructType, str],
- is_barrier: bool = False,
+ barrier: bool = False,
) -> "DataFrame":
- return self._map_partitions(
- func, schema, PythonEvalType.SQL_MAP_PANDAS_ITER_UDF, is_barrier
- )
+ return self._map_partitions(func, schema, PythonEvalType.SQL_MAP_PANDAS_ITER_UDF, barrier)
mapInPandas.__doc__ = PySparkDataFrame.mapInPandas.__doc__
@@ -1681,9 +1679,9 @@ class DataFrame:
self,
func: "ArrowMapIterFunction",
schema: Union[StructType, str],
- is_barrier: bool = False,
+ barrier: bool = False,
) -> "DataFrame":
- return self._map_partitions(func, schema, PythonEvalType.SQL_MAP_ARROW_ITER_UDF, is_barrier)
+ return self._map_partitions(func, schema, PythonEvalType.SQL_MAP_ARROW_ITER_UDF, barrier)
mapInArrow.__doc__ = PySparkDataFrame.mapInArrow.__doc__
diff --git a/python/pyspark/sql/connect/proto/relations_pb2.pyi b/python/pyspark/sql/connect/proto/relations_pb2.pyi
index c3cf733a995..cd2b0fb2d3f 100644
--- a/python/pyspark/sql/connect/proto/relations_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/relations_pb2.pyi
@@ -2767,7 +2767,7 @@ class MapPartitions(google.protobuf.message.Message):
def func(self) -> pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction:
"""(Required) Input user-defined function."""
is_barrier: builtins.bool
- """(Optional) isBarrier."""
+ """(Optional) Whether to use barrier mode execution or not."""
def __init__(
self,
*,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org