You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2024/12/12 02:24:30 UTC
(spark) branch master updated: [SPARK-50544][PYTHON][CONNECT] Implement `StructType.toDDL`
This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1f6cb60e3c78 [SPARK-50544][PYTHON][CONNECT] Implement `StructType.toDDL`
1f6cb60e3c78 is described below
commit 1f6cb60e3c78dc8592c0b76435f22d0f36194ed7
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Thu Dec 12 10:24:09 2024 +0800
[SPARK-50544][PYTHON][CONNECT] Implement `StructType.toDDL`
### What changes were proposed in this pull request?
Implement `StructType.toDDL`
### Why are the changes needed?
we has `fromDDL` in python side, but the `toDDL` is still missing
### Does this PR introduce _any_ user-facing change?
new feature
```
In [1]: spark.range(100).schema.toDDL()
Out[1]: 'id BIGINT NOT NULL'
```
### How was this patch tested?
added tests
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #49142 from zhengruifeng/struct_type_to_ddl.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
python/pyspark/sql/connect/client/core.py | 8 +
python/pyspark/sql/connect/proto/base_pb2.py | 344 +++++++++++----------
python/pyspark/sql/connect/proto/base_pb2.pyi | 47 +++
python/pyspark/sql/tests/test_types.py | 16 +
python/pyspark/sql/types.py | 19 ++
.../src/main/protobuf/spark/connect/base.proto | 11 +
.../service/SparkConnectAnalyzeHandler.scala | 12 +
.../spark/sql/api/python/PythonSQLUtils.scala | 4 +
8 files changed, 291 insertions(+), 170 deletions(-)
diff --git a/python/pyspark/sql/connect/client/core.py b/python/pyspark/sql/connect/client/core.py
index 78d4e0fc1c4f..af24af90a3dc 100644
--- a/python/pyspark/sql/connect/client/core.py
+++ b/python/pyspark/sql/connect/client/core.py
@@ -494,6 +494,7 @@ class AnalyzeResult:
is_same_semantics: Optional[bool],
semantic_hash: Optional[int],
storage_level: Optional[StorageLevel],
+ ddl_string: Optional[str],
):
self.schema = schema
self.explain_string = explain_string
@@ -506,6 +507,7 @@ class AnalyzeResult:
self.is_same_semantics = is_same_semantics
self.semantic_hash = semantic_hash
self.storage_level = storage_level
+ self.ddl_string = ddl_string
@classmethod
def fromProto(cls, pb: Any) -> "AnalyzeResult":
@@ -520,6 +522,7 @@ class AnalyzeResult:
is_same_semantics: Optional[bool] = None
semantic_hash: Optional[int] = None
storage_level: Optional[StorageLevel] = None
+ ddl_string: Optional[str] = None
if pb.HasField("schema"):
schema = types.proto_schema_to_pyspark_data_type(pb.schema.schema)
@@ -547,6 +550,8 @@ class AnalyzeResult:
pass
elif pb.HasField("get_storage_level"):
storage_level = proto_to_storage_level(pb.get_storage_level.storage_level)
+ elif pb.HasField("json_to_ddl"):
+ ddl_string = pb.json_to_ddl.ddl_string
else:
raise SparkConnectException("No analyze result found!")
@@ -562,6 +567,7 @@ class AnalyzeResult:
is_same_semantics,
semantic_hash,
storage_level,
+ ddl_string,
)
@@ -1284,6 +1290,8 @@ class SparkConnectClient(object):
req.unpersist.blocking = cast(bool, kwargs.get("blocking"))
elif method == "get_storage_level":
req.get_storage_level.relation.CopyFrom(cast(pb2.Relation, kwargs.get("relation")))
+ elif method == "json_to_ddl":
+ req.json_to_ddl.json_string = cast(str, kwargs.get("json_string"))
else:
raise PySparkValueError(
errorClass="UNSUPPORTED_OPERATION",
diff --git a/python/pyspark/sql/connect/proto/base_pb2.py b/python/pyspark/sql/connect/proto/base_pb2.py
index 620f413f62c0..97694c33abeb 100644
--- a/python/pyspark/sql/connect/proto/base_pb2.py
+++ b/python/pyspark/sql/connect/proto/base_pb2.py
@@ -43,7 +43,7 @@ from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
- b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"t\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"z\n\x0bUserContext\x12\x17 [...]
+ b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"t\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"z\n\x0bUserContext\x12\x17 [...]
)
_globals = globals()
@@ -73,181 +73,185 @@ if not _descriptor._USE_C_DESCRIPTORS:
_globals["_USERCONTEXT"]._serialized_start = 337
_globals["_USERCONTEXT"]._serialized_end = 459
_globals["_ANALYZEPLANREQUEST"]._serialized_start = 462
- _globals["_ANALYZEPLANREQUEST"]._serialized_end = 3014
- _globals["_ANALYZEPLANREQUEST_SCHEMA"]._serialized_start = 1745
- _globals["_ANALYZEPLANREQUEST_SCHEMA"]._serialized_end = 1794
- _globals["_ANALYZEPLANREQUEST_EXPLAIN"]._serialized_start = 1797
- _globals["_ANALYZEPLANREQUEST_EXPLAIN"]._serialized_end = 2112
- _globals["_ANALYZEPLANREQUEST_EXPLAIN_EXPLAINMODE"]._serialized_start = 1940
- _globals["_ANALYZEPLANREQUEST_EXPLAIN_EXPLAINMODE"]._serialized_end = 2112
- _globals["_ANALYZEPLANREQUEST_TREESTRING"]._serialized_start = 2114
- _globals["_ANALYZEPLANREQUEST_TREESTRING"]._serialized_end = 2204
- _globals["_ANALYZEPLANREQUEST_ISLOCAL"]._serialized_start = 2206
- _globals["_ANALYZEPLANREQUEST_ISLOCAL"]._serialized_end = 2256
- _globals["_ANALYZEPLANREQUEST_ISSTREAMING"]._serialized_start = 2258
- _globals["_ANALYZEPLANREQUEST_ISSTREAMING"]._serialized_end = 2312
- _globals["_ANALYZEPLANREQUEST_INPUTFILES"]._serialized_start = 2314
- _globals["_ANALYZEPLANREQUEST_INPUTFILES"]._serialized_end = 2367
- _globals["_ANALYZEPLANREQUEST_SPARKVERSION"]._serialized_start = 2369
- _globals["_ANALYZEPLANREQUEST_SPARKVERSION"]._serialized_end = 2383
- _globals["_ANALYZEPLANREQUEST_DDLPARSE"]._serialized_start = 2385
- _globals["_ANALYZEPLANREQUEST_DDLPARSE"]._serialized_end = 2426
- _globals["_ANALYZEPLANREQUEST_SAMESEMANTICS"]._serialized_start = 2428
- _globals["_ANALYZEPLANREQUEST_SAMESEMANTICS"]._serialized_end = 2549
- _globals["_ANALYZEPLANREQUEST_SEMANTICHASH"]._serialized_start = 2551
- _globals["_ANALYZEPLANREQUEST_SEMANTICHASH"]._serialized_end = 2606
- _globals["_ANALYZEPLANREQUEST_PERSIST"]._serialized_start = 2609
- _globals["_ANALYZEPLANREQUEST_PERSIST"]._serialized_end = 2760
- _globals["_ANALYZEPLANREQUEST_UNPERSIST"]._serialized_start = 2762
- _globals["_ANALYZEPLANREQUEST_UNPERSIST"]._serialized_end = 2872
- _globals["_ANALYZEPLANREQUEST_GETSTORAGELEVEL"]._serialized_start = 2874
- _globals["_ANALYZEPLANREQUEST_GETSTORAGELEVEL"]._serialized_end = 2944
- _globals["_ANALYZEPLANRESPONSE"]._serialized_start = 3017
- _globals["_ANALYZEPLANRESPONSE"]._serialized_end = 4759
- _globals["_ANALYZEPLANRESPONSE_SCHEMA"]._serialized_start = 4178
- _globals["_ANALYZEPLANRESPONSE_SCHEMA"]._serialized_end = 4235
- _globals["_ANALYZEPLANRESPONSE_EXPLAIN"]._serialized_start = 4237
- _globals["_ANALYZEPLANRESPONSE_EXPLAIN"]._serialized_end = 4285
- _globals["_ANALYZEPLANRESPONSE_TREESTRING"]._serialized_start = 4287
- _globals["_ANALYZEPLANRESPONSE_TREESTRING"]._serialized_end = 4332
- _globals["_ANALYZEPLANRESPONSE_ISLOCAL"]._serialized_start = 4334
- _globals["_ANALYZEPLANRESPONSE_ISLOCAL"]._serialized_end = 4370
- _globals["_ANALYZEPLANRESPONSE_ISSTREAMING"]._serialized_start = 4372
- _globals["_ANALYZEPLANRESPONSE_ISSTREAMING"]._serialized_end = 4420
- _globals["_ANALYZEPLANRESPONSE_INPUTFILES"]._serialized_start = 4422
- _globals["_ANALYZEPLANRESPONSE_INPUTFILES"]._serialized_end = 4456
- _globals["_ANALYZEPLANRESPONSE_SPARKVERSION"]._serialized_start = 4458
- _globals["_ANALYZEPLANRESPONSE_SPARKVERSION"]._serialized_end = 4498
- _globals["_ANALYZEPLANRESPONSE_DDLPARSE"]._serialized_start = 4500
- _globals["_ANALYZEPLANRESPONSE_DDLPARSE"]._serialized_end = 4559
- _globals["_ANALYZEPLANRESPONSE_SAMESEMANTICS"]._serialized_start = 4561
- _globals["_ANALYZEPLANRESPONSE_SAMESEMANTICS"]._serialized_end = 4600
- _globals["_ANALYZEPLANRESPONSE_SEMANTICHASH"]._serialized_start = 4602
- _globals["_ANALYZEPLANRESPONSE_SEMANTICHASH"]._serialized_end = 4640
- _globals["_ANALYZEPLANRESPONSE_PERSIST"]._serialized_start = 2609
- _globals["_ANALYZEPLANRESPONSE_PERSIST"]._serialized_end = 2618
- _globals["_ANALYZEPLANRESPONSE_UNPERSIST"]._serialized_start = 2762
- _globals["_ANALYZEPLANRESPONSE_UNPERSIST"]._serialized_end = 2773
- _globals["_ANALYZEPLANRESPONSE_GETSTORAGELEVEL"]._serialized_start = 4666
- _globals["_ANALYZEPLANRESPONSE_GETSTORAGELEVEL"]._serialized_end = 4749
- _globals["_EXECUTEPLANREQUEST"]._serialized_start = 4762
- _globals["_EXECUTEPLANREQUEST"]._serialized_end = 5437
- _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_start = 5196
- _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_end = 5361
- _globals["_EXECUTEPLANRESPONSE"]._serialized_start = 5440
- _globals["_EXECUTEPLANRESPONSE"]._serialized_end = 8358
- _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_start = 7132
- _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_end = 7203
- _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_start = 7205
- _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_end = 7323
- _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_start = 7326
- _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_end = 7843
- _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_start = 7421
- _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_end = 7753
+ _globals["_ANALYZEPLANREQUEST"]._serialized_end = 3139
+ _globals["_ANALYZEPLANREQUEST_SCHEMA"]._serialized_start = 1824
+ _globals["_ANALYZEPLANREQUEST_SCHEMA"]._serialized_end = 1873
+ _globals["_ANALYZEPLANREQUEST_EXPLAIN"]._serialized_start = 1876
+ _globals["_ANALYZEPLANREQUEST_EXPLAIN"]._serialized_end = 2191
+ _globals["_ANALYZEPLANREQUEST_EXPLAIN_EXPLAINMODE"]._serialized_start = 2019
+ _globals["_ANALYZEPLANREQUEST_EXPLAIN_EXPLAINMODE"]._serialized_end = 2191
+ _globals["_ANALYZEPLANREQUEST_TREESTRING"]._serialized_start = 2193
+ _globals["_ANALYZEPLANREQUEST_TREESTRING"]._serialized_end = 2283
+ _globals["_ANALYZEPLANREQUEST_ISLOCAL"]._serialized_start = 2285
+ _globals["_ANALYZEPLANREQUEST_ISLOCAL"]._serialized_end = 2335
+ _globals["_ANALYZEPLANREQUEST_ISSTREAMING"]._serialized_start = 2337
+ _globals["_ANALYZEPLANREQUEST_ISSTREAMING"]._serialized_end = 2391
+ _globals["_ANALYZEPLANREQUEST_INPUTFILES"]._serialized_start = 2393
+ _globals["_ANALYZEPLANREQUEST_INPUTFILES"]._serialized_end = 2446
+ _globals["_ANALYZEPLANREQUEST_SPARKVERSION"]._serialized_start = 2448
+ _globals["_ANALYZEPLANREQUEST_SPARKVERSION"]._serialized_end = 2462
+ _globals["_ANALYZEPLANREQUEST_DDLPARSE"]._serialized_start = 2464
+ _globals["_ANALYZEPLANREQUEST_DDLPARSE"]._serialized_end = 2505
+ _globals["_ANALYZEPLANREQUEST_SAMESEMANTICS"]._serialized_start = 2507
+ _globals["_ANALYZEPLANREQUEST_SAMESEMANTICS"]._serialized_end = 2628
+ _globals["_ANALYZEPLANREQUEST_SEMANTICHASH"]._serialized_start = 2630
+ _globals["_ANALYZEPLANREQUEST_SEMANTICHASH"]._serialized_end = 2685
+ _globals["_ANALYZEPLANREQUEST_PERSIST"]._serialized_start = 2688
+ _globals["_ANALYZEPLANREQUEST_PERSIST"]._serialized_end = 2839
+ _globals["_ANALYZEPLANREQUEST_UNPERSIST"]._serialized_start = 2841
+ _globals["_ANALYZEPLANREQUEST_UNPERSIST"]._serialized_end = 2951
+ _globals["_ANALYZEPLANREQUEST_GETSTORAGELEVEL"]._serialized_start = 2953
+ _globals["_ANALYZEPLANREQUEST_GETSTORAGELEVEL"]._serialized_end = 3023
+ _globals["_ANALYZEPLANREQUEST_JSONTODDL"]._serialized_start = 3025
+ _globals["_ANALYZEPLANREQUEST_JSONTODDL"]._serialized_end = 3069
+ _globals["_ANALYZEPLANRESPONSE"]._serialized_start = 3142
+ _globals["_ANALYZEPLANRESPONSE"]._serialized_end = 5008
+ _globals["_ANALYZEPLANRESPONSE_SCHEMA"]._serialized_start = 4383
+ _globals["_ANALYZEPLANRESPONSE_SCHEMA"]._serialized_end = 4440
+ _globals["_ANALYZEPLANRESPONSE_EXPLAIN"]._serialized_start = 4442
+ _globals["_ANALYZEPLANRESPONSE_EXPLAIN"]._serialized_end = 4490
+ _globals["_ANALYZEPLANRESPONSE_TREESTRING"]._serialized_start = 4492
+ _globals["_ANALYZEPLANRESPONSE_TREESTRING"]._serialized_end = 4537
+ _globals["_ANALYZEPLANRESPONSE_ISLOCAL"]._serialized_start = 4539
+ _globals["_ANALYZEPLANRESPONSE_ISLOCAL"]._serialized_end = 4575
+ _globals["_ANALYZEPLANRESPONSE_ISSTREAMING"]._serialized_start = 4577
+ _globals["_ANALYZEPLANRESPONSE_ISSTREAMING"]._serialized_end = 4625
+ _globals["_ANALYZEPLANRESPONSE_INPUTFILES"]._serialized_start = 4627
+ _globals["_ANALYZEPLANRESPONSE_INPUTFILES"]._serialized_end = 4661
+ _globals["_ANALYZEPLANRESPONSE_SPARKVERSION"]._serialized_start = 4663
+ _globals["_ANALYZEPLANRESPONSE_SPARKVERSION"]._serialized_end = 4703
+ _globals["_ANALYZEPLANRESPONSE_DDLPARSE"]._serialized_start = 4705
+ _globals["_ANALYZEPLANRESPONSE_DDLPARSE"]._serialized_end = 4764
+ _globals["_ANALYZEPLANRESPONSE_SAMESEMANTICS"]._serialized_start = 4766
+ _globals["_ANALYZEPLANRESPONSE_SAMESEMANTICS"]._serialized_end = 4805
+ _globals["_ANALYZEPLANRESPONSE_SEMANTICHASH"]._serialized_start = 4807
+ _globals["_ANALYZEPLANRESPONSE_SEMANTICHASH"]._serialized_end = 4845
+ _globals["_ANALYZEPLANRESPONSE_PERSIST"]._serialized_start = 2688
+ _globals["_ANALYZEPLANRESPONSE_PERSIST"]._serialized_end = 2697
+ _globals["_ANALYZEPLANRESPONSE_UNPERSIST"]._serialized_start = 2841
+ _globals["_ANALYZEPLANRESPONSE_UNPERSIST"]._serialized_end = 2852
+ _globals["_ANALYZEPLANRESPONSE_GETSTORAGELEVEL"]._serialized_start = 4871
+ _globals["_ANALYZEPLANRESPONSE_GETSTORAGELEVEL"]._serialized_end = 4954
+ _globals["_ANALYZEPLANRESPONSE_JSONTODDL"]._serialized_start = 4956
+ _globals["_ANALYZEPLANRESPONSE_JSONTODDL"]._serialized_end = 4998
+ _globals["_EXECUTEPLANREQUEST"]._serialized_start = 5011
+ _globals["_EXECUTEPLANREQUEST"]._serialized_end = 5686
+ _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_start = 5445
+ _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_end = 5610
+ _globals["_EXECUTEPLANRESPONSE"]._serialized_start = 5689
+ _globals["_EXECUTEPLANRESPONSE"]._serialized_end = 8607
+ _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_start = 7381
+ _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_end = 7452
+ _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_start = 7454
+ _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_end = 7572
+ _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_start = 7575
+ _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_end = 8092
+ _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_start = 7670
+ _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_end = 8002
_globals[
"_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY"
- ]._serialized_start = 7630
+ ]._serialized_start = 7879
_globals[
"_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY"
- ]._serialized_end = 7753
- _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_start = 7755
- _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_end = 7843
- _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_start = 7846
- _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_end = 7987
- _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_start = 7989
- _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_end = 8005
- _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_start = 8008
- _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_end = 8341
- _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_start = 8164
- _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_end = 8341
- _globals["_KEYVALUE"]._serialized_start = 8360
- _globals["_KEYVALUE"]._serialized_end = 8425
- _globals["_CONFIGREQUEST"]._serialized_start = 8428
- _globals["_CONFIGREQUEST"]._serialized_end = 9587
- _globals["_CONFIGREQUEST_OPERATION"]._serialized_start = 8736
- _globals["_CONFIGREQUEST_OPERATION"]._serialized_end = 9234
- _globals["_CONFIGREQUEST_SET"]._serialized_start = 9236
- _globals["_CONFIGREQUEST_SET"]._serialized_end = 9288
- _globals["_CONFIGREQUEST_GET"]._serialized_start = 9290
- _globals["_CONFIGREQUEST_GET"]._serialized_end = 9315
- _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_start = 9317
- _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_end = 9380
- _globals["_CONFIGREQUEST_GETOPTION"]._serialized_start = 9382
- _globals["_CONFIGREQUEST_GETOPTION"]._serialized_end = 9413
- _globals["_CONFIGREQUEST_GETALL"]._serialized_start = 9415
- _globals["_CONFIGREQUEST_GETALL"]._serialized_end = 9463
- _globals["_CONFIGREQUEST_UNSET"]._serialized_start = 9465
- _globals["_CONFIGREQUEST_UNSET"]._serialized_end = 9492
- _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_start = 9494
- _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_end = 9528
- _globals["_CONFIGRESPONSE"]._serialized_start = 9590
- _globals["_CONFIGRESPONSE"]._serialized_end = 9765
- _globals["_ADDARTIFACTSREQUEST"]._serialized_start = 9768
- _globals["_ADDARTIFACTSREQUEST"]._serialized_end = 10770
- _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_start = 10243
- _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_end = 10296
- _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_start = 10298
- _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_end = 10409
- _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_start = 10411
- _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_end = 10504
- _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_start = 10507
- _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_end = 10700
- _globals["_ADDARTIFACTSRESPONSE"]._serialized_start = 10773
- _globals["_ADDARTIFACTSRESPONSE"]._serialized_end = 11045
- _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_start = 10964
- _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_end = 11045
- _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_start = 11048
- _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_end = 11374
- _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_start = 11377
- _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_end = 11729
- _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_start = 11572
- _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_end = 11687
- _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_start = 11689
- _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_end = 11729
- _globals["_INTERRUPTREQUEST"]._serialized_start = 11732
- _globals["_INTERRUPTREQUEST"]._serialized_end = 12335
- _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_start = 12135
- _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_end = 12263
- _globals["_INTERRUPTRESPONSE"]._serialized_start = 12338
- _globals["_INTERRUPTRESPONSE"]._serialized_end = 12482
- _globals["_REATTACHOPTIONS"]._serialized_start = 12484
- _globals["_REATTACHOPTIONS"]._serialized_end = 12537
- _globals["_REATTACHEXECUTEREQUEST"]._serialized_start = 12540
- _globals["_REATTACHEXECUTEREQUEST"]._serialized_end = 12946
- _globals["_RELEASEEXECUTEREQUEST"]._serialized_start = 12949
- _globals["_RELEASEEXECUTEREQUEST"]._serialized_end = 13534
- _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_start = 13403
- _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_end = 13415
- _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_start = 13417
- _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_end = 13464
- _globals["_RELEASEEXECUTERESPONSE"]._serialized_start = 13537
- _globals["_RELEASEEXECUTERESPONSE"]._serialized_end = 13702
- _globals["_RELEASESESSIONREQUEST"]._serialized_start = 13705
- _globals["_RELEASESESSIONREQUEST"]._serialized_end = 13876
- _globals["_RELEASESESSIONRESPONSE"]._serialized_start = 13878
- _globals["_RELEASESESSIONRESPONSE"]._serialized_end = 13986
- _globals["_FETCHERRORDETAILSREQUEST"]._serialized_start = 13989
- _globals["_FETCHERRORDETAILSREQUEST"]._serialized_end = 14321
- _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_start = 14324
- _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_end = 15879
- _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_start = 14553
- _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_end = 14727
- _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_start = 14730
- _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_end = 15098
- _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_start = 15061
- _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_end = 15098
- _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_start = 15101
- _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_end = 15510
+ ]._serialized_end = 8002
+ _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_start = 8004
+ _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_end = 8092
+ _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_start = 8095
+ _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_end = 8236
+ _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_start = 8238
+ _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_end = 8254
+ _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_start = 8257
+ _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_end = 8590
+ _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_start = 8413
+ _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_end = 8590
+ _globals["_KEYVALUE"]._serialized_start = 8609
+ _globals["_KEYVALUE"]._serialized_end = 8674
+ _globals["_CONFIGREQUEST"]._serialized_start = 8677
+ _globals["_CONFIGREQUEST"]._serialized_end = 9836
+ _globals["_CONFIGREQUEST_OPERATION"]._serialized_start = 8985
+ _globals["_CONFIGREQUEST_OPERATION"]._serialized_end = 9483
+ _globals["_CONFIGREQUEST_SET"]._serialized_start = 9485
+ _globals["_CONFIGREQUEST_SET"]._serialized_end = 9537
+ _globals["_CONFIGREQUEST_GET"]._serialized_start = 9539
+ _globals["_CONFIGREQUEST_GET"]._serialized_end = 9564
+ _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_start = 9566
+ _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_end = 9629
+ _globals["_CONFIGREQUEST_GETOPTION"]._serialized_start = 9631
+ _globals["_CONFIGREQUEST_GETOPTION"]._serialized_end = 9662
+ _globals["_CONFIGREQUEST_GETALL"]._serialized_start = 9664
+ _globals["_CONFIGREQUEST_GETALL"]._serialized_end = 9712
+ _globals["_CONFIGREQUEST_UNSET"]._serialized_start = 9714
+ _globals["_CONFIGREQUEST_UNSET"]._serialized_end = 9741
+ _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_start = 9743
+ _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_end = 9777
+ _globals["_CONFIGRESPONSE"]._serialized_start = 9839
+ _globals["_CONFIGRESPONSE"]._serialized_end = 10014
+ _globals["_ADDARTIFACTSREQUEST"]._serialized_start = 10017
+ _globals["_ADDARTIFACTSREQUEST"]._serialized_end = 11019
+ _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_start = 10492
+ _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_end = 10545
+ _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_start = 10547
+ _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_end = 10658
+ _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_start = 10660
+ _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_end = 10753
+ _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_start = 10756
+ _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_end = 10949
+ _globals["_ADDARTIFACTSRESPONSE"]._serialized_start = 11022
+ _globals["_ADDARTIFACTSRESPONSE"]._serialized_end = 11294
+ _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_start = 11213
+ _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_end = 11294
+ _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_start = 11297
+ _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_end = 11623
+ _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_start = 11626
+ _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_end = 11978
+ _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_start = 11821
+ _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_end = 11936
+ _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_start = 11938
+ _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_end = 11978
+ _globals["_INTERRUPTREQUEST"]._serialized_start = 11981
+ _globals["_INTERRUPTREQUEST"]._serialized_end = 12584
+ _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_start = 12384
+ _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_end = 12512
+ _globals["_INTERRUPTRESPONSE"]._serialized_start = 12587
+ _globals["_INTERRUPTRESPONSE"]._serialized_end = 12731
+ _globals["_REATTACHOPTIONS"]._serialized_start = 12733
+ _globals["_REATTACHOPTIONS"]._serialized_end = 12786
+ _globals["_REATTACHEXECUTEREQUEST"]._serialized_start = 12789
+ _globals["_REATTACHEXECUTEREQUEST"]._serialized_end = 13195
+ _globals["_RELEASEEXECUTEREQUEST"]._serialized_start = 13198
+ _globals["_RELEASEEXECUTEREQUEST"]._serialized_end = 13783
+ _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_start = 13652
+ _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_end = 13664
+ _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_start = 13666
+ _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_end = 13713
+ _globals["_RELEASEEXECUTERESPONSE"]._serialized_start = 13786
+ _globals["_RELEASEEXECUTERESPONSE"]._serialized_end = 13951
+ _globals["_RELEASESESSIONREQUEST"]._serialized_start = 13954
+ _globals["_RELEASESESSIONREQUEST"]._serialized_end = 14125
+ _globals["_RELEASESESSIONRESPONSE"]._serialized_start = 14127
+ _globals["_RELEASESESSIONRESPONSE"]._serialized_end = 14235
+ _globals["_FETCHERRORDETAILSREQUEST"]._serialized_start = 14238
+ _globals["_FETCHERRORDETAILSREQUEST"]._serialized_end = 14570
+ _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_start = 14573
+ _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_end = 16128
+ _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_start = 14802
+ _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_end = 14976
+ _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_start = 14979
+ _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_end = 15347
+ _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_start = 15310
+ _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_end = 15347
+ _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_start = 15350
+ _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_end = 15759
_globals[
"_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"
- ]._serialized_start = 15412
+ ]._serialized_start = 15661
_globals[
"_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"
- ]._serialized_end = 15480
- _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_start = 15513
- _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_end = 15860
- _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_start = 15881
- _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_end = 15971
- _globals["_SPARKCONNECTSERVICE"]._serialized_start = 15974
- _globals["_SPARKCONNECTSERVICE"]._serialized_end = 16920
+ ]._serialized_end = 15729
+ _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_start = 15762
+ _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_end = 16109
+ _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_start = 16130
+ _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_end = 16220
+ _globals["_SPARKCONNECTSERVICE"]._serialized_start = 16223
+ _globals["_SPARKCONNECTSERVICE"]._serialized_end = 17169
# @@protoc_insertion_point(module_scope)
diff --git a/python/pyspark/sql/connect/proto/base_pb2.pyi b/python/pyspark/sql/connect/proto/base_pb2.pyi
index 5db25569828b..253f8a58166a 100644
--- a/python/pyspark/sql/connect/proto/base_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/base_pb2.pyi
@@ -477,6 +477,21 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
self, field_name: typing_extensions.Literal["relation", b"relation"]
) -> None: ...
+ class JsonToDDL(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ JSON_STRING_FIELD_NUMBER: builtins.int
+ json_string: builtins.str
+ """(Required) The JSON formatted string to be converted to DDL."""
+ def __init__(
+ self,
+ *,
+ json_string: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(
+ self, field_name: typing_extensions.Literal["json_string", b"json_string"]
+ ) -> None: ...
+
SESSION_ID_FIELD_NUMBER: builtins.int
CLIENT_OBSERVED_SERVER_SIDE_SESSION_ID_FIELD_NUMBER: builtins.int
USER_CONTEXT_FIELD_NUMBER: builtins.int
@@ -494,6 +509,7 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
PERSIST_FIELD_NUMBER: builtins.int
UNPERSIST_FIELD_NUMBER: builtins.int
GET_STORAGE_LEVEL_FIELD_NUMBER: builtins.int
+ JSON_TO_DDL_FIELD_NUMBER: builtins.int
session_id: builtins.str
"""(Required)
@@ -542,6 +558,8 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
def unpersist(self) -> global___AnalyzePlanRequest.Unpersist: ...
@property
def get_storage_level(self) -> global___AnalyzePlanRequest.GetStorageLevel: ...
+ @property
+ def json_to_ddl(self) -> global___AnalyzePlanRequest.JsonToDDL: ...
def __init__(
self,
*,
@@ -562,6 +580,7 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
persist: global___AnalyzePlanRequest.Persist | None = ...,
unpersist: global___AnalyzePlanRequest.Unpersist | None = ...,
get_storage_level: global___AnalyzePlanRequest.GetStorageLevel | None = ...,
+ json_to_ddl: global___AnalyzePlanRequest.JsonToDDL | None = ...,
) -> None: ...
def HasField(
self,
@@ -588,6 +607,8 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
b"is_local",
"is_streaming",
b"is_streaming",
+ "json_to_ddl",
+ b"json_to_ddl",
"persist",
b"persist",
"same_semantics",
@@ -631,6 +652,8 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
b"is_local",
"is_streaming",
b"is_streaming",
+ "json_to_ddl",
+ b"json_to_ddl",
"persist",
b"persist",
"same_semantics",
@@ -680,6 +703,7 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
"persist",
"unpersist",
"get_storage_level",
+ "json_to_ddl",
]
| None
): ...
@@ -877,6 +901,20 @@ class AnalyzePlanResponse(google.protobuf.message.Message):
self, field_name: typing_extensions.Literal["storage_level", b"storage_level"]
) -> None: ...
+ class JsonToDDL(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+ DDL_STRING_FIELD_NUMBER: builtins.int
+ ddl_string: builtins.str
+ def __init__(
+ self,
+ *,
+ ddl_string: builtins.str = ...,
+ ) -> None: ...
+ def ClearField(
+ self, field_name: typing_extensions.Literal["ddl_string", b"ddl_string"]
+ ) -> None: ...
+
SESSION_ID_FIELD_NUMBER: builtins.int
SERVER_SIDE_SESSION_ID_FIELD_NUMBER: builtins.int
SCHEMA_FIELD_NUMBER: builtins.int
@@ -892,6 +930,7 @@ class AnalyzePlanResponse(google.protobuf.message.Message):
PERSIST_FIELD_NUMBER: builtins.int
UNPERSIST_FIELD_NUMBER: builtins.int
GET_STORAGE_LEVEL_FIELD_NUMBER: builtins.int
+ JSON_TO_DDL_FIELD_NUMBER: builtins.int
session_id: builtins.str
server_side_session_id: builtins.str
"""Server-side generated idempotency key that the client can use to assert that the server side
@@ -923,6 +962,8 @@ class AnalyzePlanResponse(google.protobuf.message.Message):
def unpersist(self) -> global___AnalyzePlanResponse.Unpersist: ...
@property
def get_storage_level(self) -> global___AnalyzePlanResponse.GetStorageLevel: ...
+ @property
+ def json_to_ddl(self) -> global___AnalyzePlanResponse.JsonToDDL: ...
def __init__(
self,
*,
@@ -941,6 +982,7 @@ class AnalyzePlanResponse(google.protobuf.message.Message):
persist: global___AnalyzePlanResponse.Persist | None = ...,
unpersist: global___AnalyzePlanResponse.Unpersist | None = ...,
get_storage_level: global___AnalyzePlanResponse.GetStorageLevel | None = ...,
+ json_to_ddl: global___AnalyzePlanResponse.JsonToDDL | None = ...,
) -> None: ...
def HasField(
self,
@@ -957,6 +999,8 @@ class AnalyzePlanResponse(google.protobuf.message.Message):
b"is_local",
"is_streaming",
b"is_streaming",
+ "json_to_ddl",
+ b"json_to_ddl",
"persist",
b"persist",
"result",
@@ -990,6 +1034,8 @@ class AnalyzePlanResponse(google.protobuf.message.Message):
b"is_local",
"is_streaming",
b"is_streaming",
+ "json_to_ddl",
+ b"json_to_ddl",
"persist",
b"persist",
"result",
@@ -1029,6 +1075,7 @@ class AnalyzePlanResponse(google.protobuf.message.Message):
"persist",
"unpersist",
"get_storage_level",
+ "json_to_ddl",
]
| None
): ...
diff --git a/python/pyspark/sql/tests/test_types.py b/python/pyspark/sql/tests/test_types.py
index dcc383b7add5..432ddd083c80 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -2240,6 +2240,22 @@ class TypesTestsMixin:
PySparkValueError, lambda: str(VariantVal(bytes([32, 10, 1, 0, 0, 0]), metadata))
)
+ def test_to_ddl(self):
+ schema = StructType().add("a", NullType()).add("b", BooleanType()).add("c", BinaryType())
+ self.assertEqual(schema.toDDL(), "a VOID,b BOOLEAN,c BINARY")
+
+ schema = StructType().add("a", IntegerType()).add("b", StringType())
+ self.assertEqual(schema.toDDL(), "a INT,b STRING")
+
+ schema = StructType().add("a", FloatType()).add("b", LongType(), False)
+ self.assertEqual(schema.toDDL(), "a FLOAT,b BIGINT NOT NULL")
+
+ schema = StructType().add("a", ArrayType(DoubleType()), False).add("b", DateType())
+ self.assertEqual(schema.toDDL(), "a ARRAY<DOUBLE> NOT NULL,b DATE")
+
+ schema = StructType().add("a", TimestampType()).add("b", TimestampNTZType())
+ self.assertEqual(schema.toDDL(), "a TIMESTAMP,b TIMESTAMP_NTZ")
+
def test_from_ddl(self):
self.assertEqual(DataType.fromDDL("long"), LongType())
self.assertEqual(
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 03227c8c8760..303c0c3c8bd0 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -1555,6 +1555,25 @@ class StructType(DataType):
field._build_formatted_string(prefix, stringConcat, depth)
return stringConcat.toString()
+ def toDDL(self) -> str:
+ from pyspark.sql.utils import is_remote
+
+ if is_remote():
+ from pyspark.sql.connect.session import SparkSession
+
+ session = SparkSession.getActiveSession()
+ assert session is not None
+ return session._client._analyze( # type: ignore[return-value]
+ method="json_to_ddl", json_string=self.json()
+ ).ddl_string
+
+ else:
+ from py4j.java_gateway import JVMView
+
+ sc = get_active_spark_context()
+ assert sc._jvm is not None
+ return cast(JVMView, sc._jvm).PythonSQLUtils.jsonToDDL(self.json())
+
class VariantType(AtomicType):
"""
diff --git a/sql/connect/common/src/main/protobuf/spark/connect/base.proto b/sql/connect/common/src/main/protobuf/spark/connect/base.proto
index e3c84ddd5e88..e27049d2114d 100644
--- a/sql/connect/common/src/main/protobuf/spark/connect/base.proto
+++ b/sql/connect/common/src/main/protobuf/spark/connect/base.proto
@@ -94,6 +94,7 @@ message AnalyzePlanRequest {
Persist persist = 14;
Unpersist unpersist = 15;
GetStorageLevel get_storage_level = 16;
+ JsonToDDL json_to_ddl = 18;
}
message Schema {
@@ -199,6 +200,11 @@ message AnalyzePlanRequest {
// (Required) The logical plan to get the storage level.
Relation relation = 1;
}
+
+ message JsonToDDL {
+ // (Required) The JSON formatted string to be converted to DDL.
+ string json_string = 1;
+ }
}
// Response to performing analysis of the query. Contains relevant metadata to be able to
@@ -224,6 +230,7 @@ message AnalyzePlanResponse {
Persist persist = 12;
Unpersist unpersist = 13;
GetStorageLevel get_storage_level = 14;
+ JsonToDDL json_to_ddl = 16;
}
message Schema {
@@ -275,6 +282,10 @@ message AnalyzePlanResponse {
// (Required) The StorageLevel as a result of get_storage_level request.
StorageLevel storage_level = 1;
}
+
+ message JsonToDDL {
+ string ddl_string = 1;
+ }
}
// A request to be executed by the service.
diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectAnalyzeHandler.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectAnalyzeHandler.scala
index 6c5d95ac67d3..8ca021c5be39 100644
--- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectAnalyzeHandler.scala
+++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectAnalyzeHandler.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.Dataset
import org.apache.spark.sql.connect.common.{DataTypeProtoConverter, InvalidPlanInput, StorageLevelProtoConverter}
import org.apache.spark.sql.connect.planner.SparkConnectPlanner
import org.apache.spark.sql.execution.{CodegenMode, CostMode, ExtendedMode, FormattedMode, SimpleMode}
+import org.apache.spark.sql.types.{DataType, StructType}
import org.apache.spark.util.ArrayImplicits._
private[connect] class SparkConnectAnalyzeHandler(
@@ -206,6 +207,17 @@ private[connect] class SparkConnectAnalyzeHandler(
.setStorageLevel(StorageLevelProtoConverter.toConnectProtoType(storageLevel))
.build())
+ case proto.AnalyzePlanRequest.AnalyzeCase.JSON_TO_DDL =>
+ val ddl = DataType
+ .fromJson(request.getJsonToDdl.getJsonString)
+ .asInstanceOf[StructType]
+ .toDDL
+ builder.setJsonToDdl(
+ proto.AnalyzePlanResponse.JsonToDDL
+ .newBuilder()
+ .setDdlString(ddl)
+ .build())
+
case other => throw InvalidPlanInput(s"Unknown Analyze Method $other!")
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
index da03293ce743..e33fe38b160a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
@@ -144,6 +144,10 @@ private[sql] object PythonSQLUtils extends Logging {
}
}
+ def jsonToDDL(json: String): String = {
+ DataType.fromJson(json).asInstanceOf[StructType].toDDL
+ }
+
def unresolvedNamedLambdaVariable(name: String): Column =
Column(internal.UnresolvedNamedLambdaVariable.apply(name))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org