You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by hv...@apache.org on 2023/05/10 18:47:39 UTC
[spark] branch master updated: [SPARK-43430][CONNECT][PROTO] ExecutePlanRequest supports arbitrary request options
This is an automated email from the ASF dual-hosted git repository.
hvanhovell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 4c4b53c85a1 [SPARK-43430][CONNECT][PROTO] ExecutePlanRequest supports arbitrary request options
4c4b53c85a1 is described below
commit 4c4b53c85a188689e8df4228937a6e899d88e638
Author: Martin Grund <ma...@databricks.com>
AuthorDate: Wed May 10 14:47:21 2023 -0400
[SPARK-43430][CONNECT][PROTO] ExecutePlanRequest supports arbitrary request options
### What changes were proposed in this pull request?
This is an extension to the `ExecutePlanRequest` proto message to support arbitrary request options. Possible use cases are pagination or result formats.
For example, if we were to support additional result serialization types the client could indicate them by adding a simple message to the request options.
```
{
request_options : [
request_option : {
result_format: {
type: "csv"
}
}
]
}
```
### Why are the changes needed?
Extensibility
### Does this PR introduce _any_ user-facing change?
No
Closes #41110 from grundprinzip/SPARK-43430.
Authored-by: Martin Grund <ma...@databricks.com>
Signed-off-by: Herman van Hovell <he...@databricks.com>
---
.../src/main/protobuf/spark/connect/base.proto | 11 ++
python/pyspark/sql/connect/proto/base_pb2.py | 153 +++++++++++----------
python/pyspark/sql/connect/proto/base_pb2.pyi | 42 ++++++
3 files changed, 136 insertions(+), 70 deletions(-)
diff --git a/connector/connect/common/src/main/protobuf/spark/connect/base.proto b/connector/connect/common/src/main/protobuf/spark/connect/base.proto
index b304a8aea63..f54e28e3b61 100644
--- a/connector/connect/common/src/main/protobuf/spark/connect/base.proto
+++ b/connector/connect/common/src/main/protobuf/spark/connect/base.proto
@@ -287,6 +287,17 @@ message ExecutePlanRequest {
// can be used for language or version specific information and is only intended for
// logging purposes and will not be interpreted by the server.
optional string client_type = 4;
+
+ // Repeated element for options that can be passed to the request. This element is currently
+ // unused but allows to pass in an extension value used for arbitrary options.
+ repeated RequestOption request_options = 5;
+
+ message RequestOption {
+ oneof request_option {
+ // Extension type for request options
+ google.protobuf.Any extension = 999;
+ }
+ }
}
// The response of a query, can be one or more for each request. Responses belonging to the
diff --git a/python/pyspark/sql/connect/proto/base_pb2.py b/python/pyspark/sql/connect/proto/base_pb2.py
index 997f59911da..55ee830bfd4 100644
--- a/python/pyspark/sql/connect/proto/base_pb2.py
+++ b/python/pyspark/sql/connect/proto/base_pb2.py
@@ -38,7 +38,7 @@ from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
- b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"t\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"z\n\x0bUserContext\x12\x17 [...]
+ b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"t\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"z\n\x0bUserContext\x12\x17 [...]
)
@@ -73,6 +73,7 @@ _ANALYZEPLANRESPONSE_PERSIST = _ANALYZEPLANRESPONSE.nested_types_by_name["Persis
_ANALYZEPLANRESPONSE_UNPERSIST = _ANALYZEPLANRESPONSE.nested_types_by_name["Unpersist"]
_ANALYZEPLANRESPONSE_GETSTORAGELEVEL = _ANALYZEPLANRESPONSE.nested_types_by_name["GetStorageLevel"]
_EXECUTEPLANREQUEST = DESCRIPTOR.message_types_by_name["ExecutePlanRequest"]
+_EXECUTEPLANREQUEST_REQUESTOPTION = _EXECUTEPLANREQUEST.nested_types_by_name["RequestOption"]
_EXECUTEPLANRESPONSE = DESCRIPTOR.message_types_by_name["ExecutePlanResponse"]
_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT = _EXECUTEPLANRESPONSE.nested_types_by_name[
"SqlCommandResult"
@@ -435,12 +436,22 @@ ExecutePlanRequest = _reflection.GeneratedProtocolMessageType(
"ExecutePlanRequest",
(_message.Message,),
{
+ "RequestOption": _reflection.GeneratedProtocolMessageType(
+ "RequestOption",
+ (_message.Message,),
+ {
+ "DESCRIPTOR": _EXECUTEPLANREQUEST_REQUESTOPTION,
+ "__module__": "spark.connect.base_pb2"
+ # @@protoc_insertion_point(class_scope:spark.connect.ExecutePlanRequest.RequestOption)
+ },
+ ),
"DESCRIPTOR": _EXECUTEPLANREQUEST,
"__module__": "spark.connect.base_pb2"
# @@protoc_insertion_point(class_scope:spark.connect.ExecutePlanRequest)
},
)
_sym_db.RegisterMessage(ExecutePlanRequest)
+_sym_db.RegisterMessage(ExecutePlanRequest.RequestOption)
ExecutePlanResponse = _reflection.GeneratedProtocolMessageType(
"ExecutePlanResponse",
@@ -844,73 +855,75 @@ if _descriptor._USE_C_DESCRIPTORS == False:
_ANALYZEPLANRESPONSE_GETSTORAGELEVEL._serialized_start = 4482
_ANALYZEPLANRESPONSE_GETSTORAGELEVEL._serialized_end = 4565
_EXECUTEPLANREQUEST._serialized_start = 4578
- _EXECUTEPLANREQUEST._serialized_end = 4787
- _EXECUTEPLANRESPONSE._serialized_start = 4790
- _EXECUTEPLANRESPONSE._serialized_end = 6555
- _EXECUTEPLANRESPONSE_SQLCOMMANDRESULT._serialized_start = 5786
- _EXECUTEPLANRESPONSE_SQLCOMMANDRESULT._serialized_end = 5857
- _EXECUTEPLANRESPONSE_ARROWBATCH._serialized_start = 5859
- _EXECUTEPLANRESPONSE_ARROWBATCH._serialized_end = 5920
- _EXECUTEPLANRESPONSE_METRICS._serialized_start = 5923
- _EXECUTEPLANRESPONSE_METRICS._serialized_end = 6440
- _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT._serialized_start = 6018
- _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT._serialized_end = 6350
- _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY._serialized_start = 6227
- _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY._serialized_end = 6350
- _EXECUTEPLANRESPONSE_METRICS_METRICVALUE._serialized_start = 6352
- _EXECUTEPLANRESPONSE_METRICS_METRICVALUE._serialized_end = 6440
- _EXECUTEPLANRESPONSE_OBSERVEDMETRICS._serialized_start = 6442
- _EXECUTEPLANRESPONSE_OBSERVEDMETRICS._serialized_end = 6538
- _KEYVALUE._serialized_start = 6557
- _KEYVALUE._serialized_end = 6622
- _CONFIGREQUEST._serialized_start = 6625
- _CONFIGREQUEST._serialized_end = 7653
- _CONFIGREQUEST_OPERATION._serialized_start = 6845
- _CONFIGREQUEST_OPERATION._serialized_end = 7343
- _CONFIGREQUEST_SET._serialized_start = 7345
- _CONFIGREQUEST_SET._serialized_end = 7397
- _CONFIGREQUEST_GET._serialized_start = 7399
- _CONFIGREQUEST_GET._serialized_end = 7424
- _CONFIGREQUEST_GETWITHDEFAULT._serialized_start = 7426
- _CONFIGREQUEST_GETWITHDEFAULT._serialized_end = 7489
- _CONFIGREQUEST_GETOPTION._serialized_start = 7491
- _CONFIGREQUEST_GETOPTION._serialized_end = 7522
- _CONFIGREQUEST_GETALL._serialized_start = 7524
- _CONFIGREQUEST_GETALL._serialized_end = 7572
- _CONFIGREQUEST_UNSET._serialized_start = 7574
- _CONFIGREQUEST_UNSET._serialized_end = 7601
- _CONFIGREQUEST_ISMODIFIABLE._serialized_start = 7603
- _CONFIGREQUEST_ISMODIFIABLE._serialized_end = 7637
- _CONFIGRESPONSE._serialized_start = 7655
- _CONFIGRESPONSE._serialized_end = 7777
- _ADDARTIFACTSREQUEST._serialized_start = 7780
- _ADDARTIFACTSREQUEST._serialized_end = 8651
- _ADDARTIFACTSREQUEST_ARTIFACTCHUNK._serialized_start = 8167
- _ADDARTIFACTSREQUEST_ARTIFACTCHUNK._serialized_end = 8220
- _ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT._serialized_start = 8222
- _ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT._serialized_end = 8333
- _ADDARTIFACTSREQUEST_BATCH._serialized_start = 8335
- _ADDARTIFACTSREQUEST_BATCH._serialized_end = 8428
- _ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT._serialized_start = 8431
- _ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT._serialized_end = 8624
- _ADDARTIFACTSRESPONSE._serialized_start = 8654
- _ADDARTIFACTSRESPONSE._serialized_end = 8842
- _ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY._serialized_start = 8761
- _ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY._serialized_end = 8842
- _ARTIFACTSTATUSESREQUEST._serialized_start = 8845
- _ARTIFACTSTATUSESREQUEST._serialized_end = 9040
- _ARTIFACTSTATUSESRESPONSE._serialized_start = 9043
- _ARTIFACTSTATUSESRESPONSE._serialized_end = 9311
- _ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS._serialized_start = 9154
- _ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS._serialized_end = 9194
- _ARTIFACTSTATUSESRESPONSE_STATUSESENTRY._serialized_start = 9196
- _ARTIFACTSTATUSESRESPONSE_STATUSESENTRY._serialized_end = 9311
- _INTERRUPTREQUEST._serialized_start = 9314
- _INTERRUPTREQUEST._serialized_end = 9639
- _INTERRUPTREQUEST_INTERRUPTTYPE._serialized_start = 9552
- _INTERRUPTREQUEST_INTERRUPTTYPE._serialized_end = 9623
- _INTERRUPTRESPONSE._serialized_start = 9641
- _INTERRUPTRESPONSE._serialized_end = 9691
- _SPARKCONNECTSERVICE._serialized_start = 9694
- _SPARKCONNECTSERVICE._serialized_end = 10242
+ _EXECUTEPLANREQUEST._serialized_end = 4967
+ _EXECUTEPLANREQUEST_REQUESTOPTION._serialized_start = 4863
+ _EXECUTEPLANREQUEST_REQUESTOPTION._serialized_end = 4951
+ _EXECUTEPLANRESPONSE._serialized_start = 4970
+ _EXECUTEPLANRESPONSE._serialized_end = 6735
+ _EXECUTEPLANRESPONSE_SQLCOMMANDRESULT._serialized_start = 5966
+ _EXECUTEPLANRESPONSE_SQLCOMMANDRESULT._serialized_end = 6037
+ _EXECUTEPLANRESPONSE_ARROWBATCH._serialized_start = 6039
+ _EXECUTEPLANRESPONSE_ARROWBATCH._serialized_end = 6100
+ _EXECUTEPLANRESPONSE_METRICS._serialized_start = 6103
+ _EXECUTEPLANRESPONSE_METRICS._serialized_end = 6620
+ _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT._serialized_start = 6198
+ _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT._serialized_end = 6530
+ _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY._serialized_start = 6407
+ _EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY._serialized_end = 6530
+ _EXECUTEPLANRESPONSE_METRICS_METRICVALUE._serialized_start = 6532
+ _EXECUTEPLANRESPONSE_METRICS_METRICVALUE._serialized_end = 6620
+ _EXECUTEPLANRESPONSE_OBSERVEDMETRICS._serialized_start = 6622
+ _EXECUTEPLANRESPONSE_OBSERVEDMETRICS._serialized_end = 6718
+ _KEYVALUE._serialized_start = 6737
+ _KEYVALUE._serialized_end = 6802
+ _CONFIGREQUEST._serialized_start = 6805
+ _CONFIGREQUEST._serialized_end = 7833
+ _CONFIGREQUEST_OPERATION._serialized_start = 7025
+ _CONFIGREQUEST_OPERATION._serialized_end = 7523
+ _CONFIGREQUEST_SET._serialized_start = 7525
+ _CONFIGREQUEST_SET._serialized_end = 7577
+ _CONFIGREQUEST_GET._serialized_start = 7579
+ _CONFIGREQUEST_GET._serialized_end = 7604
+ _CONFIGREQUEST_GETWITHDEFAULT._serialized_start = 7606
+ _CONFIGREQUEST_GETWITHDEFAULT._serialized_end = 7669
+ _CONFIGREQUEST_GETOPTION._serialized_start = 7671
+ _CONFIGREQUEST_GETOPTION._serialized_end = 7702
+ _CONFIGREQUEST_GETALL._serialized_start = 7704
+ _CONFIGREQUEST_GETALL._serialized_end = 7752
+ _CONFIGREQUEST_UNSET._serialized_start = 7754
+ _CONFIGREQUEST_UNSET._serialized_end = 7781
+ _CONFIGREQUEST_ISMODIFIABLE._serialized_start = 7783
+ _CONFIGREQUEST_ISMODIFIABLE._serialized_end = 7817
+ _CONFIGRESPONSE._serialized_start = 7835
+ _CONFIGRESPONSE._serialized_end = 7957
+ _ADDARTIFACTSREQUEST._serialized_start = 7960
+ _ADDARTIFACTSREQUEST._serialized_end = 8831
+ _ADDARTIFACTSREQUEST_ARTIFACTCHUNK._serialized_start = 8347
+ _ADDARTIFACTSREQUEST_ARTIFACTCHUNK._serialized_end = 8400
+ _ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT._serialized_start = 8402
+ _ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT._serialized_end = 8513
+ _ADDARTIFACTSREQUEST_BATCH._serialized_start = 8515
+ _ADDARTIFACTSREQUEST_BATCH._serialized_end = 8608
+ _ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT._serialized_start = 8611
+ _ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT._serialized_end = 8804
+ _ADDARTIFACTSRESPONSE._serialized_start = 8834
+ _ADDARTIFACTSRESPONSE._serialized_end = 9022
+ _ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY._serialized_start = 8941
+ _ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY._serialized_end = 9022
+ _ARTIFACTSTATUSESREQUEST._serialized_start = 9025
+ _ARTIFACTSTATUSESREQUEST._serialized_end = 9220
+ _ARTIFACTSTATUSESRESPONSE._serialized_start = 9223
+ _ARTIFACTSTATUSESRESPONSE._serialized_end = 9491
+ _ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS._serialized_start = 9334
+ _ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS._serialized_end = 9374
+ _ARTIFACTSTATUSESRESPONSE_STATUSESENTRY._serialized_start = 9376
+ _ARTIFACTSTATUSESRESPONSE_STATUSESENTRY._serialized_end = 9491
+ _INTERRUPTREQUEST._serialized_start = 9494
+ _INTERRUPTREQUEST._serialized_end = 9819
+ _INTERRUPTREQUEST_INTERRUPTTYPE._serialized_start = 9732
+ _INTERRUPTREQUEST_INTERRUPTTYPE._serialized_end = 9803
+ _INTERRUPTRESPONSE._serialized_start = 9821
+ _INTERRUPTRESPONSE._serialized_end = 9871
+ _SPARKCONNECTSERVICE._serialized_start = 9874
+ _SPARKCONNECTSERVICE._serialized_end = 10422
# @@protoc_insertion_point(module_scope)
diff --git a/python/pyspark/sql/connect/proto/base_pb2.pyi b/python/pyspark/sql/connect/proto/base_pb2.pyi
index fb1bf92c3c3..633058f33ed 100644
--- a/python/pyspark/sql/connect/proto/base_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/base_pb2.pyi
@@ -1001,10 +1001,39 @@ class ExecutePlanRequest(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
+ class RequestOption(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ EXTENSION_FIELD_NUMBER: builtins.int
+ @property
+ def extension(self) -> google.protobuf.any_pb2.Any:
+ """Extension type for request options"""
+ def __init__(
+ self,
+ *,
+ extension: google.protobuf.any_pb2.Any | None = ...,
+ ) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "extension", b"extension", "request_option", b"request_option"
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "extension", b"extension", "request_option", b"request_option"
+ ],
+ ) -> None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["request_option", b"request_option"]
+ ) -> typing_extensions.Literal["extension"] | None: ...
+
SESSION_ID_FIELD_NUMBER: builtins.int
USER_CONTEXT_FIELD_NUMBER: builtins.int
PLAN_FIELD_NUMBER: builtins.int
CLIENT_TYPE_FIELD_NUMBER: builtins.int
+ REQUEST_OPTIONS_FIELD_NUMBER: builtins.int
session_id: builtins.str
"""(Required)
@@ -1027,6 +1056,15 @@ class ExecutePlanRequest(google.protobuf.message.Message):
can be used for language or version specific information and is only intended for
logging purposes and will not be interpreted by the server.
"""
+ @property
+ def request_options(
+ self,
+ ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[
+ global___ExecutePlanRequest.RequestOption
+ ]:
+ """Repeated element for options that can be passed to the request. This element is currently
+ unused but allows to pass in an extension value used for arbitrary options.
+ """
def __init__(
self,
*,
@@ -1034,6 +1072,8 @@ class ExecutePlanRequest(google.protobuf.message.Message):
user_context: global___UserContext | None = ...,
plan: global___Plan | None = ...,
client_type: builtins.str | None = ...,
+ request_options: collections.abc.Iterable[global___ExecutePlanRequest.RequestOption]
+ | None = ...,
) -> None: ...
def HasField(
self,
@@ -1057,6 +1097,8 @@ class ExecutePlanRequest(google.protobuf.message.Message):
b"client_type",
"plan",
b"plan",
+ "request_options",
+ b"request_options",
"session_id",
b"session_id",
"user_context",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org