You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2023/03/09 08:50:05 UTC
[spark] branch branch-3.4 updated: [SPARK-42710][CONNECT][PYTHON] Rename FrameMap proto to MapPartitions
This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new f3e69a1fe21 [SPARK-42710][CONNECT][PYTHON] Rename FrameMap proto to MapPartitions
f3e69a1fe21 is described below
commit f3e69a1fe219e8a796c3fd862286a95fa9d0ae91
Author: Xinrong Meng <xi...@apache.org>
AuthorDate: Thu Mar 9 16:49:21 2023 +0800
[SPARK-42710][CONNECT][PYTHON] Rename FrameMap proto to MapPartitions
### What changes were proposed in this pull request?
Rename FrameMap proto to MapPartitions.
### Why are the changes needed?
For readability.
Frame Map API refers to mapInPandas and mapInArrow, which are equivalent to MapPartitions.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing tests.
Closes #40329 from xinrong-meng/mapInArrow.
Authored-by: Xinrong Meng <xi...@apache.org>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
(cherry picked from commit 7a67be1f7ca4c7a56910b4a395565f563f4b9c1c)
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
.../main/protobuf/spark/connect/relations.proto | 8 +-
.../sql/connect/planner/SparkConnectPlanner.scala | 6 +-
python/pyspark/sql/connect/dataframe.py | 2 +-
python/pyspark/sql/connect/plan.py | 8 +-
python/pyspark/sql/connect/proto/relations_pb2.py | 240 ++++++++++-----------
python/pyspark/sql/connect/proto/relations_pb2.pyi | 24 +--
6 files changed, 144 insertions(+), 144 deletions(-)
diff --git a/connector/connect/common/src/main/protobuf/spark/connect/relations.proto b/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
index 97fc3a474f3..69451e7b76e 100644
--- a/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
+++ b/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
@@ -60,7 +60,7 @@ message Relation {
Unpivot unpivot = 25;
ToSchema to_schema = 26;
RepartitionByExpression repartition_by_expression = 27;
- FrameMap frame_map = 28;
+ MapPartitions map_partitions = 28;
CollectMetrics collect_metrics = 29;
Parse parse = 30;
@@ -780,11 +780,11 @@ message RepartitionByExpression {
optional int32 num_partitions = 3;
}
-message FrameMap {
- // (Required) Input relation for a Frame Map API: mapInPandas, mapInArrow.
+message MapPartitions {
+ // (Required) Input relation for a mapPartitions-equivalent API: mapInPandas, mapInArrow.
Relation input = 1;
- // (Required) Input user-defined function of a Frame Map API.
+ // (Required) Input user-defined function.
CommonInlineUserDefinedFunction func = 2;
}
diff --git a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
index 9a8402a1e98..cd4da39d62f 100644
--- a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
+++ b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
@@ -114,8 +114,8 @@ class SparkConnectPlanner(val session: SparkSession) {
case proto.Relation.RelTypeCase.UNPIVOT => transformUnpivot(rel.getUnpivot)
case proto.Relation.RelTypeCase.REPARTITION_BY_EXPRESSION =>
transformRepartitionByExpression(rel.getRepartitionByExpression)
- case proto.Relation.RelTypeCase.FRAME_MAP =>
- transformFrameMap(rel.getFrameMap)
+ case proto.Relation.RelTypeCase.MAP_PARTITIONS =>
+ transformMapPartitions(rel.getMapPartitions)
case proto.Relation.RelTypeCase.COLLECT_METRICS =>
transformCollectMetrics(rel.getCollectMetrics)
case proto.Relation.RelTypeCase.PARSE => transformParse(rel.getParse)
@@ -471,7 +471,7 @@ class SparkConnectPlanner(val session: SparkSession) {
.logicalPlan
}
- private def transformFrameMap(rel: proto.FrameMap): LogicalPlan = {
+ private def transformMapPartitions(rel: proto.MapPartitions): LogicalPlan = {
val commonUdf = rel.getFunc
val pythonUdf = transformPythonUDF(commonUdf)
pythonUdf.evalType match {
diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py
index 504b83d1165..69921896f46 100644
--- a/python/pyspark/sql/connect/dataframe.py
+++ b/python/pyspark/sql/connect/dataframe.py
@@ -1585,7 +1585,7 @@ class DataFrame:
)
return DataFrame.withPlan(
- plan.FrameMap(child=self._plan, function=udf_obj, cols=self.columns),
+ plan.MapPartitions(child=self._plan, function=udf_obj, cols=self.columns),
session=self._session,
)
diff --git a/python/pyspark/sql/connect/plan.py b/python/pyspark/sql/connect/plan.py
index 7444cd14c18..8e4db63dc7e 100644
--- a/python/pyspark/sql/connect/plan.py
+++ b/python/pyspark/sql/connect/plan.py
@@ -1901,8 +1901,8 @@ class ListCatalogs(LogicalPlan):
return proto.Relation(catalog=proto.Catalog(list_catalogs=proto.ListCatalogs()))
-class FrameMap(LogicalPlan):
- """Logical plan object for a Frame Map API: mapInPandas, mapInArrow."""
+class MapPartitions(LogicalPlan):
+ """Logical plan object for a mapPartitions-equivalent API: mapInPandas, mapInArrow."""
def __init__(
self, child: Optional["LogicalPlan"], function: "UserDefinedFunction", cols: List[str]
@@ -1914,8 +1914,8 @@ class FrameMap(LogicalPlan):
def plan(self, session: "SparkConnectClient") -> proto.Relation:
assert self._child is not None
plan = self._create_proto_relation()
- plan.frame_map.input.CopyFrom(self._child.plan(session))
- plan.frame_map.func.CopyFrom(self._func.to_plan_udf(session))
+ plan.map_partitions.input.CopyFrom(self._child.plan(session))
+ plan.map_partitions.func.CopyFrom(self._func.to_plan_udf(session))
return plan
diff --git a/python/pyspark/sql/connect/proto/relations_pb2.py b/python/pyspark/sql/connect/proto/relations_pb2.py
index 81fa3916c5a..521a10f214c 100644
--- a/python/pyspark/sql/connect/proto/relations_pb2.py
+++ b/python/pyspark/sql/connect/proto/relations_pb2.py
@@ -36,7 +36,7 @@ from pyspark.sql.connect.proto import catalog_pb2 as spark_dot_connect_dot_catal
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
- b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x19spark/connect/types.proto\x1a\x1bspark/connect/catalog.proto"\xa9\x13\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66il [...]
+ b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x19spark/connect/types.proto\x1a\x1bspark/connect/catalog.proto"\xb8\x13\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66il [...]
)
@@ -91,7 +91,7 @@ _UNPIVOT = DESCRIPTOR.message_types_by_name["Unpivot"]
_UNPIVOT_VALUES = _UNPIVOT.nested_types_by_name["Values"]
_TOSCHEMA = DESCRIPTOR.message_types_by_name["ToSchema"]
_REPARTITIONBYEXPRESSION = DESCRIPTOR.message_types_by_name["RepartitionByExpression"]
-_FRAMEMAP = DESCRIPTOR.message_types_by_name["FrameMap"]
+_MAPPARTITIONS = DESCRIPTOR.message_types_by_name["MapPartitions"]
_COLLECTMETRICS = DESCRIPTOR.message_types_by_name["CollectMetrics"]
_PARSE = DESCRIPTOR.message_types_by_name["Parse"]
_PARSE_OPTIONSENTRY = _PARSE.nested_types_by_name["OptionsEntry"]
@@ -629,16 +629,16 @@ RepartitionByExpression = _reflection.GeneratedProtocolMessageType(
)
_sym_db.RegisterMessage(RepartitionByExpression)
-FrameMap = _reflection.GeneratedProtocolMessageType(
- "FrameMap",
+MapPartitions = _reflection.GeneratedProtocolMessageType(
+ "MapPartitions",
(_message.Message,),
{
- "DESCRIPTOR": _FRAMEMAP,
+ "DESCRIPTOR": _MAPPARTITIONS,
"__module__": "spark.connect.relations_pb2"
- # @@protoc_insertion_point(class_scope:spark.connect.FrameMap)
+ # @@protoc_insertion_point(class_scope:spark.connect.MapPartitions)
},
)
-_sym_db.RegisterMessage(FrameMap)
+_sym_db.RegisterMessage(MapPartitions)
CollectMetrics = _reflection.GeneratedProtocolMessageType(
"CollectMetrics",
@@ -685,117 +685,117 @@ if _descriptor._USE_C_DESCRIPTORS == False:
_PARSE_OPTIONSENTRY._options = None
_PARSE_OPTIONSENTRY._serialized_options = b"8\001"
_RELATION._serialized_start = 165
- _RELATION._serialized_end = 2638
- _UNKNOWN._serialized_start = 2640
- _UNKNOWN._serialized_end = 2649
- _RELATIONCOMMON._serialized_start = 2651
- _RELATIONCOMMON._serialized_end = 2742
- _SQL._serialized_start = 2745
- _SQL._serialized_end = 2879
- _SQL_ARGSENTRY._serialized_start = 2824
- _SQL_ARGSENTRY._serialized_end = 2879
- _READ._serialized_start = 2882
- _READ._serialized_end = 3378
- _READ_NAMEDTABLE._serialized_start = 3024
- _READ_NAMEDTABLE._serialized_end = 3085
- _READ_DATASOURCE._serialized_start = 3088
- _READ_DATASOURCE._serialized_end = 3365
- _READ_DATASOURCE_OPTIONSENTRY._serialized_start = 3285
- _READ_DATASOURCE_OPTIONSENTRY._serialized_end = 3343
- _PROJECT._serialized_start = 3380
- _PROJECT._serialized_end = 3497
- _FILTER._serialized_start = 3499
- _FILTER._serialized_end = 3611
- _JOIN._serialized_start = 3614
- _JOIN._serialized_end = 4085
- _JOIN_JOINTYPE._serialized_start = 3877
- _JOIN_JOINTYPE._serialized_end = 4085
- _SETOPERATION._serialized_start = 4088
- _SETOPERATION._serialized_end = 4567
- _SETOPERATION_SETOPTYPE._serialized_start = 4404
- _SETOPERATION_SETOPTYPE._serialized_end = 4518
- _LIMIT._serialized_start = 4569
- _LIMIT._serialized_end = 4645
- _OFFSET._serialized_start = 4647
- _OFFSET._serialized_end = 4726
- _TAIL._serialized_start = 4728
- _TAIL._serialized_end = 4803
- _AGGREGATE._serialized_start = 4806
- _AGGREGATE._serialized_end = 5388
- _AGGREGATE_PIVOT._serialized_start = 5145
- _AGGREGATE_PIVOT._serialized_end = 5256
- _AGGREGATE_GROUPTYPE._serialized_start = 5259
- _AGGREGATE_GROUPTYPE._serialized_end = 5388
- _SORT._serialized_start = 5391
- _SORT._serialized_end = 5551
- _DROP._serialized_start = 5554
- _DROP._serialized_end = 5695
- _DEDUPLICATE._serialized_start = 5698
- _DEDUPLICATE._serialized_end = 5869
- _LOCALRELATION._serialized_start = 5871
- _LOCALRELATION._serialized_end = 5960
- _SAMPLE._serialized_start = 5963
- _SAMPLE._serialized_end = 6236
- _RANGE._serialized_start = 6239
- _RANGE._serialized_end = 6384
- _SUBQUERYALIAS._serialized_start = 6386
- _SUBQUERYALIAS._serialized_end = 6500
- _REPARTITION._serialized_start = 6503
- _REPARTITION._serialized_end = 6645
- _SHOWSTRING._serialized_start = 6648
- _SHOWSTRING._serialized_end = 6790
- _STATSUMMARY._serialized_start = 6792
- _STATSUMMARY._serialized_end = 6884
- _STATDESCRIBE._serialized_start = 6886
- _STATDESCRIBE._serialized_end = 6967
- _STATCROSSTAB._serialized_start = 6969
- _STATCROSSTAB._serialized_end = 7070
- _STATCOV._serialized_start = 7072
- _STATCOV._serialized_end = 7168
- _STATCORR._serialized_start = 7171
- _STATCORR._serialized_end = 7308
- _STATAPPROXQUANTILE._serialized_start = 7311
- _STATAPPROXQUANTILE._serialized_end = 7475
- _STATFREQITEMS._serialized_start = 7477
- _STATFREQITEMS._serialized_end = 7602
- _STATSAMPLEBY._serialized_start = 7605
- _STATSAMPLEBY._serialized_end = 7914
- _STATSAMPLEBY_FRACTION._serialized_start = 7806
- _STATSAMPLEBY_FRACTION._serialized_end = 7905
- _NAFILL._serialized_start = 7917
- _NAFILL._serialized_end = 8051
- _NADROP._serialized_start = 8054
- _NADROP._serialized_end = 8188
- _NAREPLACE._serialized_start = 8191
- _NAREPLACE._serialized_end = 8487
- _NAREPLACE_REPLACEMENT._serialized_start = 8346
- _NAREPLACE_REPLACEMENT._serialized_end = 8487
- _TODF._serialized_start = 8489
- _TODF._serialized_end = 8577
- _WITHCOLUMNSRENAMED._serialized_start = 8580
- _WITHCOLUMNSRENAMED._serialized_end = 8819
- _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_start = 8752
- _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_end = 8819
- _WITHCOLUMNS._serialized_start = 8821
- _WITHCOLUMNS._serialized_end = 8940
- _HINT._serialized_start = 8943
- _HINT._serialized_end = 9075
- _UNPIVOT._serialized_start = 9078
- _UNPIVOT._serialized_end = 9405
- _UNPIVOT_VALUES._serialized_start = 9335
- _UNPIVOT_VALUES._serialized_end = 9394
- _TOSCHEMA._serialized_start = 9407
- _TOSCHEMA._serialized_end = 9513
- _REPARTITIONBYEXPRESSION._serialized_start = 9516
- _REPARTITIONBYEXPRESSION._serialized_end = 9719
- _FRAMEMAP._serialized_start = 9721
- _FRAMEMAP._serialized_end = 9846
- _COLLECTMETRICS._serialized_start = 9849
- _COLLECTMETRICS._serialized_end = 9985
- _PARSE._serialized_start = 9988
- _PARSE._serialized_end = 10376
- _PARSE_OPTIONSENTRY._serialized_start = 3285
- _PARSE_OPTIONSENTRY._serialized_end = 3343
- _PARSE_PARSEFORMAT._serialized_start = 10277
- _PARSE_PARSEFORMAT._serialized_end = 10365
+ _RELATION._serialized_end = 2653
+ _UNKNOWN._serialized_start = 2655
+ _UNKNOWN._serialized_end = 2664
+ _RELATIONCOMMON._serialized_start = 2666
+ _RELATIONCOMMON._serialized_end = 2757
+ _SQL._serialized_start = 2760
+ _SQL._serialized_end = 2894
+ _SQL_ARGSENTRY._serialized_start = 2839
+ _SQL_ARGSENTRY._serialized_end = 2894
+ _READ._serialized_start = 2897
+ _READ._serialized_end = 3393
+ _READ_NAMEDTABLE._serialized_start = 3039
+ _READ_NAMEDTABLE._serialized_end = 3100
+ _READ_DATASOURCE._serialized_start = 3103
+ _READ_DATASOURCE._serialized_end = 3380
+ _READ_DATASOURCE_OPTIONSENTRY._serialized_start = 3300
+ _READ_DATASOURCE_OPTIONSENTRY._serialized_end = 3358
+ _PROJECT._serialized_start = 3395
+ _PROJECT._serialized_end = 3512
+ _FILTER._serialized_start = 3514
+ _FILTER._serialized_end = 3626
+ _JOIN._serialized_start = 3629
+ _JOIN._serialized_end = 4100
+ _JOIN_JOINTYPE._serialized_start = 3892
+ _JOIN_JOINTYPE._serialized_end = 4100
+ _SETOPERATION._serialized_start = 4103
+ _SETOPERATION._serialized_end = 4582
+ _SETOPERATION_SETOPTYPE._serialized_start = 4419
+ _SETOPERATION_SETOPTYPE._serialized_end = 4533
+ _LIMIT._serialized_start = 4584
+ _LIMIT._serialized_end = 4660
+ _OFFSET._serialized_start = 4662
+ _OFFSET._serialized_end = 4741
+ _TAIL._serialized_start = 4743
+ _TAIL._serialized_end = 4818
+ _AGGREGATE._serialized_start = 4821
+ _AGGREGATE._serialized_end = 5403
+ _AGGREGATE_PIVOT._serialized_start = 5160
+ _AGGREGATE_PIVOT._serialized_end = 5271
+ _AGGREGATE_GROUPTYPE._serialized_start = 5274
+ _AGGREGATE_GROUPTYPE._serialized_end = 5403
+ _SORT._serialized_start = 5406
+ _SORT._serialized_end = 5566
+ _DROP._serialized_start = 5569
+ _DROP._serialized_end = 5710
+ _DEDUPLICATE._serialized_start = 5713
+ _DEDUPLICATE._serialized_end = 5884
+ _LOCALRELATION._serialized_start = 5886
+ _LOCALRELATION._serialized_end = 5975
+ _SAMPLE._serialized_start = 5978
+ _SAMPLE._serialized_end = 6251
+ _RANGE._serialized_start = 6254
+ _RANGE._serialized_end = 6399
+ _SUBQUERYALIAS._serialized_start = 6401
+ _SUBQUERYALIAS._serialized_end = 6515
+ _REPARTITION._serialized_start = 6518
+ _REPARTITION._serialized_end = 6660
+ _SHOWSTRING._serialized_start = 6663
+ _SHOWSTRING._serialized_end = 6805
+ _STATSUMMARY._serialized_start = 6807
+ _STATSUMMARY._serialized_end = 6899
+ _STATDESCRIBE._serialized_start = 6901
+ _STATDESCRIBE._serialized_end = 6982
+ _STATCROSSTAB._serialized_start = 6984
+ _STATCROSSTAB._serialized_end = 7085
+ _STATCOV._serialized_start = 7087
+ _STATCOV._serialized_end = 7183
+ _STATCORR._serialized_start = 7186
+ _STATCORR._serialized_end = 7323
+ _STATAPPROXQUANTILE._serialized_start = 7326
+ _STATAPPROXQUANTILE._serialized_end = 7490
+ _STATFREQITEMS._serialized_start = 7492
+ _STATFREQITEMS._serialized_end = 7617
+ _STATSAMPLEBY._serialized_start = 7620
+ _STATSAMPLEBY._serialized_end = 7929
+ _STATSAMPLEBY_FRACTION._serialized_start = 7821
+ _STATSAMPLEBY_FRACTION._serialized_end = 7920
+ _NAFILL._serialized_start = 7932
+ _NAFILL._serialized_end = 8066
+ _NADROP._serialized_start = 8069
+ _NADROP._serialized_end = 8203
+ _NAREPLACE._serialized_start = 8206
+ _NAREPLACE._serialized_end = 8502
+ _NAREPLACE_REPLACEMENT._serialized_start = 8361
+ _NAREPLACE_REPLACEMENT._serialized_end = 8502
+ _TODF._serialized_start = 8504
+ _TODF._serialized_end = 8592
+ _WITHCOLUMNSRENAMED._serialized_start = 8595
+ _WITHCOLUMNSRENAMED._serialized_end = 8834
+ _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_start = 8767
+ _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_end = 8834
+ _WITHCOLUMNS._serialized_start = 8836
+ _WITHCOLUMNS._serialized_end = 8955
+ _HINT._serialized_start = 8958
+ _HINT._serialized_end = 9090
+ _UNPIVOT._serialized_start = 9093
+ _UNPIVOT._serialized_end = 9420
+ _UNPIVOT_VALUES._serialized_start = 9350
+ _UNPIVOT_VALUES._serialized_end = 9409
+ _TOSCHEMA._serialized_start = 9422
+ _TOSCHEMA._serialized_end = 9528
+ _REPARTITIONBYEXPRESSION._serialized_start = 9531
+ _REPARTITIONBYEXPRESSION._serialized_end = 9734
+ _MAPPARTITIONS._serialized_start = 9737
+ _MAPPARTITIONS._serialized_end = 9867
+ _COLLECTMETRICS._serialized_start = 9870
+ _COLLECTMETRICS._serialized_end = 10006
+ _PARSE._serialized_start = 10009
+ _PARSE._serialized_end = 10397
+ _PARSE_OPTIONSENTRY._serialized_start = 3300
+ _PARSE_OPTIONSENTRY._serialized_end = 3358
+ _PARSE_PARSEFORMAT._serialized_start = 10298
+ _PARSE_PARSEFORMAT._serialized_end = 10386
# @@protoc_insertion_point(module_scope)
diff --git a/python/pyspark/sql/connect/proto/relations_pb2.pyi b/python/pyspark/sql/connect/proto/relations_pb2.pyi
index edaab7bcb77..ab1561996ef 100644
--- a/python/pyspark/sql/connect/proto/relations_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/relations_pb2.pyi
@@ -89,7 +89,7 @@ class Relation(google.protobuf.message.Message):
UNPIVOT_FIELD_NUMBER: builtins.int
TO_SCHEMA_FIELD_NUMBER: builtins.int
REPARTITION_BY_EXPRESSION_FIELD_NUMBER: builtins.int
- FRAME_MAP_FIELD_NUMBER: builtins.int
+ MAP_PARTITIONS_FIELD_NUMBER: builtins.int
COLLECT_METRICS_FIELD_NUMBER: builtins.int
PARSE_FIELD_NUMBER: builtins.int
FILL_NA_FIELD_NUMBER: builtins.int
@@ -161,7 +161,7 @@ class Relation(google.protobuf.message.Message):
@property
def repartition_by_expression(self) -> global___RepartitionByExpression: ...
@property
- def frame_map(self) -> global___FrameMap: ...
+ def map_partitions(self) -> global___MapPartitions: ...
@property
def collect_metrics(self) -> global___CollectMetrics: ...
@property
@@ -230,7 +230,7 @@ class Relation(google.protobuf.message.Message):
unpivot: global___Unpivot | None = ...,
to_schema: global___ToSchema | None = ...,
repartition_by_expression: global___RepartitionByExpression | None = ...,
- frame_map: global___FrameMap | None = ...,
+ map_partitions: global___MapPartitions | None = ...,
collect_metrics: global___CollectMetrics | None = ...,
parse: global___Parse | None = ...,
fill_na: global___NAFill | None = ...,
@@ -281,8 +281,6 @@ class Relation(google.protobuf.message.Message):
b"fill_na",
"filter",
b"filter",
- "frame_map",
- b"frame_map",
"freq_items",
b"freq_items",
"hint",
@@ -293,6 +291,8 @@ class Relation(google.protobuf.message.Message):
b"limit",
"local_relation",
b"local_relation",
+ "map_partitions",
+ b"map_partitions",
"offset",
b"offset",
"parse",
@@ -376,8 +376,6 @@ class Relation(google.protobuf.message.Message):
b"fill_na",
"filter",
b"filter",
- "frame_map",
- b"frame_map",
"freq_items",
b"freq_items",
"hint",
@@ -388,6 +386,8 @@ class Relation(google.protobuf.message.Message):
b"limit",
"local_relation",
b"local_relation",
+ "map_partitions",
+ b"map_partitions",
"offset",
b"offset",
"parse",
@@ -467,7 +467,7 @@ class Relation(google.protobuf.message.Message):
"unpivot",
"to_schema",
"repartition_by_expression",
- "frame_map",
+ "map_partitions",
"collect_metrics",
"parse",
"fill_na",
@@ -2706,17 +2706,17 @@ class RepartitionByExpression(google.protobuf.message.Message):
global___RepartitionByExpression = RepartitionByExpression
-class FrameMap(google.protobuf.message.Message):
+class MapPartitions(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
INPUT_FIELD_NUMBER: builtins.int
FUNC_FIELD_NUMBER: builtins.int
@property
def input(self) -> global___Relation:
- """(Required) Input relation for a Frame Map API: mapInPandas, mapInArrow."""
+ """(Required) Input relation for a mapPartitions-equivalent API: mapInPandas, mapInArrow."""
@property
def func(self) -> pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction:
- """(Required) Input user-defined function of a Frame Map API."""
+ """(Required) Input user-defined function."""
def __init__(
self,
*,
@@ -2731,7 +2731,7 @@ class FrameMap(google.protobuf.message.Message):
self, field_name: typing_extensions.Literal["func", b"func", "input", b"input"]
) -> None: ...
-global___FrameMap = FrameMap
+global___MapPartitions = MapPartitions
class CollectMetrics(google.protobuf.message.Message):
"""Collect arbitrary (named) metrics from a dataset."""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org