You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2022/09/27 01:56:04 UTC
[spark] branch master updated: [SPARK-40557][CONNECT] Update generated proto files for Spark Connect
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 072575c9e6f [SPARK-40557][CONNECT] Update generated proto files for Spark Connect
072575c9e6f is described below
commit 072575c9e6fc304f09e01ad0ee180c8f309ede91
Author: Martin Grund <ma...@databricks.com>
AuthorDate: Tue Sep 27 10:55:47 2022 +0900
[SPARK-40557][CONNECT] Update generated proto files for Spark Connect
### What changes were proposed in this pull request?
This patch cleans up the generated proto files from the initial Spark Connect import. The previous files had a Databricks specific go module path embedded in the generated Python descriptor. This is now removed. No new functionality added.
### Why are the changes needed?
Cleanup.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
The generated files are used during the regular testing for Spark Connect.
Closes #37993 from grundprinzip/spark-connect-clean1.
Authored-by: Martin Grund <ma...@databricks.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
connect/src/main/buf.gen.yaml | 9 ---------
python/pyspark/sql/connect/proto/base_pb2.py | 14 ++++----------
python/pyspark/sql/connect/proto/commands_pb2.py | 6 ++----
python/pyspark/sql/connect/proto/expressions_pb2.py | 6 ++----
python/pyspark/sql/connect/proto/relations_pb2.py | 10 +++-------
python/pyspark/sql/connect/proto/types_pb2.py | 6 ++----
6 files changed, 13 insertions(+), 38 deletions(-)
diff --git a/connect/src/main/buf.gen.yaml b/connect/src/main/buf.gen.yaml
index 01e31d3c8b4..e3e15e549e8 100644
--- a/connect/src/main/buf.gen.yaml
+++ b/connect/src/main/buf.gen.yaml
@@ -26,15 +26,6 @@ plugins:
out: gen/proto/python
- remote: buf.build/grpc/plugins/python:v1.47.0-1
out: gen/proto/python
- - remote: buf.build/protocolbuffers/plugins/go:v1.28.0-1
- out: gen/proto/go
- opt:
- - paths=source_relative
- - remote: buf.build/grpc/plugins/go:v1.2.0-1
- out: gen/proto/go
- opt:
- - paths=source_relative
- - require_unimplemented_servers=false
- remote: buf.build/grpc/plugins/ruby:v1.47.0-1
out: gen/proto/ruby
- remote: buf.build/protocolbuffers/plugins/ruby:v21.2.0-1
diff --git a/python/pyspark/sql/connect/proto/base_pb2.py b/python/pyspark/sql/connect/proto/base_pb2.py
index 3adb77f77d6..910f9d644e3 100644
--- a/python/pyspark/sql/connect/proto/base_pb2.py
+++ b/python/pyspark/sql/connect/proto/base_pb2.py
@@ -28,16 +28,12 @@ from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
-from pyspark.sql.connect.proto import (
- commands_pb2 as spark_dot_connect_dot_commands__pb2,
-)
-from pyspark.sql.connect.proto import (
- relations_pb2 as spark_dot_connect_dot_relations__pb2,
-)
+from pyspark.sql.connect.proto import commands_pb2 as spark_dot_connect_dot_commands__pb2
+from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_relations__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
- b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x1cspark/connect/commands.proto\x1a\x1dspark/connect/relations.proto"t\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"\xdb\x01\n\x07Request\x12\x1b\n\tclient_id\x18\x01 \x01(\tR\x08\x63lientId\x12\x45\n\x0cuser_context\x18\x02 \x01(\x0b\x32".spark.connect.Request.UserContextR\x0buse [...]
+ b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x1cspark/connect/commands.proto\x1a\x1dspark/connect/relations.proto"t\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"\xdb\x01\n\x07Request\x12\x1b\n\tclient_id\x18\x01 \x01(\tR\x08\x63lientId\x12\x45\n\x0cuser_context\x18\x02 \x01(\x0b\x32".spark.connect.Request.UserContextR\x0buse [...]
)
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
@@ -45,9 +41,7 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "spark.connect.base_pb2", gl
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
- DESCRIPTOR._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z)github.com/databricks/spark-connect/proto"
- )
+ DESCRIPTOR._serialized_options = b"\n\036org.apache.spark.connect.protoP\001"
_RESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY._options = None
_RESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY._serialized_options = b"8\001"
_PLAN._serialized_start = 104
diff --git a/python/pyspark/sql/connect/proto/commands_pb2.py b/python/pyspark/sql/connect/proto/commands_pb2.py
index f5bb6ad5628..46d405dd008 100644
--- a/python/pyspark/sql/connect/proto/commands_pb2.py
+++ b/python/pyspark/sql/connect/proto/commands_pb2.py
@@ -32,7 +32,7 @@ from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
- b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19spark/connect/types.proto"i\n\x07\x43ommand\x12N\n\x0f\x63reate_function\x18\x01 \x01(\x0b\x32#.spark.connect.CreateScalarFunctionH\x00R\x0e\x63reateFunctionB\x0e\n\x0c\x63ommand_type"\x8f\x04\n\x14\x43reateScalarFunction\x12\x14\n\x05parts\x18\x01 \x03(\tR\x05parts\x12P\n\x08language\x18\x02 \x01(\x0e\x32\x34.spark.connect.CreateScalarFunction.FunctionLanguageR\x08language\x12\x1c\n\ttemporary\x18\x03 \x01(\x08R\ttempora [...]
+ b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19spark/connect/types.proto"i\n\x07\x43ommand\x12N\n\x0f\x63reate_function\x18\x01 \x01(\x0b\x32#.spark.connect.CreateScalarFunctionH\x00R\x0e\x63reateFunctionB\x0e\n\x0c\x63ommand_type"\x8f\x04\n\x14\x43reateScalarFunction\x12\x14\n\x05parts\x18\x01 \x03(\tR\x05parts\x12P\n\x08language\x18\x02 \x01(\x0e\x32\x34.spark.connect.CreateScalarFunction.FunctionLanguageR\x08language\x12\x1c\n\ttemporary\x18\x03 \x01(\x08R\ttempora [...]
)
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
@@ -40,9 +40,7 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "spark.connect.commands_pb2"
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
- DESCRIPTOR._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z)github.com/databricks/spark-connect/proto"
- )
+ DESCRIPTOR._serialized_options = b"\n\036org.apache.spark.connect.protoP\001"
_COMMAND._serialized_start = 74
_COMMAND._serialized_end = 179
_CREATESCALARFUNCTION._serialized_start = 182
diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.py b/python/pyspark/sql/connect/proto/expressions_pb2.py
index 16f3325d414..68a485f0a8c 100644
--- a/python/pyspark/sql/connect/proto/expressions_pb2.py
+++ b/python/pyspark/sql/connect/proto/expressions_pb2.py
@@ -33,7 +33,7 @@ from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
- b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19spark/connect/types.proto\x1a\x19google/protobuf/any.proto"\xd8\x14\n\nExpression\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolvedAttribute\x12_\n\x13unresolved_function\x18\x03 \x01(\x0b\x32,.spark.connect.Expression.UnresolvedFunctionH\x00R\x12unresolvedFu [...]
+ b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19spark/connect/types.proto\x1a\x19google/protobuf/any.proto"\xd8\x14\n\nExpression\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolvedAttribute\x12_\n\x13unresolved_function\x18\x03 \x01(\x0b\x32,.spark.connect.Expression.UnresolvedFunctionH\x00R\x12unresolvedFu [...]
)
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
@@ -41,9 +41,7 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "spark.connect.expressions_p
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
- DESCRIPTOR._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z)github.com/databricks/spark-connect/proto"
- )
+ DESCRIPTOR._serialized_options = b"\n\036org.apache.spark.connect.protoP\001"
_EXPRESSION._serialized_start = 105
_EXPRESSION._serialized_end = 2753
_EXPRESSION_LITERAL._serialized_start = 471
diff --git a/python/pyspark/sql/connect/proto/relations_pb2.py b/python/pyspark/sql/connect/proto/relations_pb2.py
index b9f74dc2380..3bfb8ddfb4f 100644
--- a/python/pyspark/sql/connect/proto/relations_pb2.py
+++ b/python/pyspark/sql/connect/proto/relations_pb2.py
@@ -28,13 +28,11 @@ from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
-from pyspark.sql.connect.proto import (
- expressions_pb2 as spark_dot_connect_dot_expressions__pb2,
-)
+from pyspark.sql.connect.proto import expressions_pb2 as spark_dot_connect_dot_expressions__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
- b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x1fspark/connect/expressions.proto"\xa6\x04\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x15.spark.connect.FilterH\x00R\x06\x66ilter\x12)\n\x04join\x18\x05 \x01(\x0 [...]
+ b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x1fspark/connect/expressions.proto"\xa6\x04\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x15.spark.connect.FilterH\x00R\x06\x66ilter\x12)\n\x04join\x18\x05 \x01(\x0 [...]
)
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
@@ -42,9 +40,7 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "spark.connect.relations_pb2
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
- DESCRIPTOR._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z)github.com/databricks/spark-connect/proto"
- )
+ DESCRIPTOR._serialized_options = b"\n\036org.apache.spark.connect.protoP\001"
_RELATION._serialized_start = 82
_RELATION._serialized_end = 632
_UNKNOWN._serialized_start = 634
diff --git a/python/pyspark/sql/connect/proto/types_pb2.py b/python/pyspark/sql/connect/proto/types_pb2.py
index 27247ca4e0c..dcaf641b855 100644
--- a/python/pyspark/sql/connect/proto/types_pb2.py
+++ b/python/pyspark/sql/connect/proto/types_pb2.py
@@ -29,7 +29,7 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
- b'\n\x19spark/connect/types.proto\x12\rspark.connect"\xea%\n\x04Type\x12\x31\n\x04\x62ool\x18\x01 \x01(\x0b\x32\x1b.spark.connect.Type.BooleanH\x00R\x04\x62ool\x12(\n\x02i8\x18\x02 \x01(\x0b\x32\x16.spark.connect.Type.I8H\x00R\x02i8\x12+\n\x03i16\x18\x03 \x01(\x0b\x32\x17.spark.connect.Type.I16H\x00R\x03i16\x12+\n\x03i32\x18\x05 \x01(\x0b\x32\x17.spark.connect.Type.I32H\x00R\x03i32\x12+\n\x03i64\x18\x07 \x01(\x0b\x32\x17.spark.connect.Type.I64H\x00R\x03i64\x12.\n\x04\x66p32\x18\n \x0 [...]
+ b'\n\x19spark/connect/types.proto\x12\rspark.connect"\xea%\n\x04Type\x12\x31\n\x04\x62ool\x18\x01 \x01(\x0b\x32\x1b.spark.connect.Type.BooleanH\x00R\x04\x62ool\x12(\n\x02i8\x18\x02 \x01(\x0b\x32\x16.spark.connect.Type.I8H\x00R\x02i8\x12+\n\x03i16\x18\x03 \x01(\x0b\x32\x17.spark.connect.Type.I16H\x00R\x03i16\x12+\n\x03i32\x18\x05 \x01(\x0b\x32\x17.spark.connect.Type.I32H\x00R\x03i32\x12+\n\x03i64\x18\x07 \x01(\x0b\x32\x17.spark.connect.Type.I64H\x00R\x03i64\x12.\n\x04\x66p32\x18\n \x0 [...]
)
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
@@ -37,9 +37,7 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "spark.connect.types_pb2", g
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
- DESCRIPTOR._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z)github.com/databricks/spark-connect/proto"
- )
+ DESCRIPTOR._serialized_options = b"\n\036org.apache.spark.connect.protoP\001"
_TYPE._serialized_start = 45
_TYPE._serialized_end = 4887
_TYPE_BOOLEAN._serialized_start = 1366
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org