You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2023/06/25 17:00:32 UTC
[spark] branch master updated: [SPARK-44178][CONNECT] Support positional parameters in `sql()`
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a903f8e9dbd [SPARK-44178][CONNECT] Support positional parameters in `sql()`
a903f8e9dbd is described below
commit a903f8e9dbd6275229cb0fa56176638533a9e371
Author: Max Gekk <ma...@gmail.com>
AuthorDate: Sun Jun 25 10:00:22 2023 -0700
[SPARK-44178][CONNECT] Support positional parameters in `sql()`
### What changes were proposed in this pull request?
In the PR, I propose to add a sequence of literal expressions to
1. Proto API: `SqlCommand` and the `SQL` relation
2. Scala connect API: `SparkSession. sql`
This PR is a follow up of https://github.com/apache/spark/pull/41568.
### Why are the changes needed?
Currently `SparkSession.sql` in Spark Connect doesn't support parameterized SQL with positional parameters. The changes allow to achieve feature parity with Scala/Java/PySpark APIs.
### Does this PR introduce _any_ user-facing change?
No, the changes just extend the existing API.
### How was this patch tested?
By running new test:
```
$ build/sbt "test:testOnly *.ClientE2ETestSuite"
```
Closes #41698 from MaxGekk/parameterized-query-pos-param-proto.
Authored-by: Max Gekk <ma...@gmail.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../scala/org/apache/spark/sql/SparkSession.scala | 37 +++-
.../org/apache/spark/sql/ClientE2ETestSuite.scala | 13 ++
.../src/main/protobuf/spark/connect/commands.proto | 3 +
.../main/protobuf/spark/connect/relations.proto | 3 +
.../sql/connect/planner/SparkConnectPlanner.scala | 20 +-
python/pyspark/sql/connect/proto/commands_pb2.py | 144 ++++++------
python/pyspark/sql/connect/proto/commands_pb2.pyi | 17 +-
python/pyspark/sql/connect/proto/relations_pb2.py | 244 ++++++++++-----------
python/pyspark/sql/connect/proto/relations_pb2.pyi | 17 +-
9 files changed, 296 insertions(+), 202 deletions(-)
diff --git a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 4cab8031978..45e7dca38d7 100644
--- a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -227,6 +227,41 @@ class SparkSession private[sql] (
createDataset(data.asScala.toSeq)
}
+ /**
+ * Executes a SQL query substituting positional parameters by the given arguments, returning the
+ * result as a `DataFrame`. This API eagerly runs DDL/DML commands, but not for SELECT queries.
+ *
+ * @param sqlText
+ * A SQL statement with positional parameters to execute.
+ * @param args
+ * An array of Java/Scala objects that can be converted to SQL literal expressions. See <a
+ * href="https://spark.apache.org/docs/latest/sql-ref-datatypes.html"> Supported Data
+ * Types</a> for supported value types in Scala/Java. For example: 1, "Steven",
+ * LocalDate.of(2023, 4, 2). A value can be also a `Column` of literal expression, in that
+ * case it is taken as is.
+ *
+ * @since 3.5.0
+ */
+ @Experimental
+ def sql(sqlText: String, args: Array[_]): DataFrame = newDataFrame { builder =>
+ // Send the SQL once to the server and then check the output.
+ val cmd = newCommand(b =>
+ b.setSqlCommand(
+ proto.SqlCommand
+ .newBuilder()
+ .setSql(sqlText)
+ .addAllPosArgs(args.map(toLiteralProto).toIterable.asJava)))
+ val plan = proto.Plan.newBuilder().setCommand(cmd)
+ val responseIter = client.execute(plan.build())
+
+ val response = responseIter.asScala
+ .find(_.hasSqlCommandResult)
+ .getOrElse(throw new RuntimeException("SQLCommandResult must be present"))
+
+ // Update the builder with the values from the result.
+ builder.mergeFrom(response.getSqlCommandResult.getRelation)
+ }
+
/**
* Executes a SQL query substituting named parameters by the given arguments, returning the
* result as a `DataFrame`. This API eagerly runs DDL/DML commands, but not for SELECT queries.
@@ -290,7 +325,7 @@ class SparkSession private[sql] (
* @since 3.4.0
*/
def sql(query: String): DataFrame = {
- sql(query, Map.empty[String, String])
+ sql(query, Array.empty)
}
/**
diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala
index 2db369273b5..b24e445964a 100644
--- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala
+++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala
@@ -947,6 +947,19 @@ class ClientE2ETestSuite extends RemoteSparkSession with SQLHelper with PrivateM
}
}
}
+
+ test("sql() with positional parameters") {
+ val result0 = spark.sql("select 1", Array.empty).collect()
+ assert(result0.length == 1 && result0(0).getInt(0) === 1)
+
+ val result1 = spark.sql("select ?", Array(1)).collect()
+ assert(result1.length == 1 && result1(0).getInt(0) === 1)
+
+ val result2 = spark.sql("select ?, ?", Array(1, "abc")).collect()
+ assert(result2.length == 1)
+ assert(result2(0).getInt(0) === 1)
+ assert(result2(0).getString(1) === "abc")
+ }
}
private[sql] case class MyType(id: Long, a: Double, b: Double)
diff --git a/connector/connect/common/src/main/protobuf/spark/connect/commands.proto b/connector/connect/common/src/main/protobuf/spark/connect/commands.proto
index 3c6bd9b0980..ac97c2dc842 100644
--- a/connector/connect/common/src/main/protobuf/spark/connect/commands.proto
+++ b/connector/connect/common/src/main/protobuf/spark/connect/commands.proto
@@ -61,6 +61,9 @@ message SqlCommand {
// (Optional) A map of parameter names to literal expressions.
map<string, Expression.Literal> args = 2;
+
+ // (Optional) A sequence of literal expressions for positional parameters in the SQL query text.
+ repeated Expression.Literal pos_args = 3;
}
// A command that can create DataFrame global temp view or local temp view.
diff --git a/connector/connect/common/src/main/protobuf/spark/connect/relations.proto b/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
index ea432bb48fc..d29ab02f86a 100644
--- a/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
+++ b/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
@@ -115,6 +115,9 @@ message SQL {
// (Optional) A map of parameter names to literal expressions.
map<string, Expression.Literal> args = 2;
+
+ // (Optional) A sequence of literal expressions for positional parameters in the SQL query text.
+ repeated Expression.Literal pos_args = 3;
}
// Relation that reads from a file / table or other data source. Does not have additional
diff --git a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
index 856d0f06ba4..907d25e1ee1 100644
--- a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
+++ b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
@@ -41,7 +41,7 @@ import org.apache.spark.ml.{functions => MLFunctions}
import org.apache.spark.sql.{Column, Dataset, Encoders, ForeachWriter, RelationalGroupedDataset, Row, SparkSession}
import org.apache.spark.sql.avro.{AvroDataToCatalyst, CatalystDataToAvro}
import org.apache.spark.sql.catalyst.{expressions, AliasIdentifier, FunctionIdentifier}
-import org.apache.spark.sql.catalyst.analysis.{GlobalTempView, LocalTempView, MultiAlias, NameParameterizedQuery, UnresolvedAlias, UnresolvedAttribute, UnresolvedDeserializer, UnresolvedExtractValue, UnresolvedFunction, UnresolvedRegex, UnresolvedRelation, UnresolvedStar}
+import org.apache.spark.sql.catalyst.analysis.{GlobalTempView, LocalTempView, MultiAlias, NameParameterizedQuery, PosParameterizedQuery, UnresolvedAlias, UnresolvedAttribute, UnresolvedDeserializer, UnresolvedExtractValue, UnresolvedFunction, UnresolvedRegex, UnresolvedRelation, UnresolvedStar}
import org.apache.spark.sql.catalyst.encoders.{AgnosticEncoder, ExpressionEncoder}
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParseException, ParserUtils}
@@ -250,10 +250,13 @@ class SparkConnectPlanner(val sessionHolder: SessionHolder) extends Logging {
private def transformSql(sql: proto.SQL): LogicalPlan = {
val args = sql.getArgsMap
+ val posArgs = sql.getPosArgsList
val parser = session.sessionState.sqlParser
val parsedPlan = parser.parsePlan(sql.getQuery)
if (!args.isEmpty) {
NameParameterizedQuery(parsedPlan, args.asScala.mapValues(transformLiteral).toMap)
+ } else if (!posArgs.isEmpty) {
+ PosParameterizedQuery(parsedPlan, posArgs.asScala.map(transformLiteral).toArray)
} else {
parsedPlan
}
@@ -2261,9 +2264,15 @@ class SparkConnectPlanner(val sessionHolder: SessionHolder) extends Logging {
sessionId: String,
responseObserver: StreamObserver[ExecutePlanResponse]): Unit = {
// Eagerly execute commands of the provided SQL string.
- val df = session.sql(
- getSqlCommand.getSql,
- getSqlCommand.getArgsMap.asScala.mapValues(transformLiteral).toMap)
+ val args = getSqlCommand.getArgsMap
+ val posArgs = getSqlCommand.getPosArgsList
+ val df = if (!args.isEmpty) {
+ session.sql(getSqlCommand.getSql, args.asScala.mapValues(transformLiteral).toMap)
+ } else if (!posArgs.isEmpty) {
+ session.sql(getSqlCommand.getSql, posArgs.asScala.map(transformLiteral).toArray)
+ } else {
+ session.sql(getSqlCommand.getSql)
+ }
// Check if commands have been executed.
val isCommand = df.queryExecution.commandExecuted.isInstanceOf[CommandResult]
val rows = df.logicalPlan match {
@@ -2317,7 +2326,8 @@ class SparkConnectPlanner(val sessionHolder: SessionHolder) extends Logging {
proto.SQL
.newBuilder()
.setQuery(getSqlCommand.getSql)
- .putAllArgs(getSqlCommand.getArgsMap)))
+ .putAllArgs(getSqlCommand.getArgsMap)
+ .addAllPosArgs(getSqlCommand.getPosArgsList)))
}
// Exactly one SQL Command Result Batch
responseObserver.onNext(
diff --git a/python/pyspark/sql/connect/proto/commands_pb2.py b/python/pyspark/sql/connect/proto/commands_pb2.py
index 689832b194a..088026d8d0b 100644
--- a/python/pyspark/sql/connect/proto/commands_pb2.py
+++ b/python/pyspark/sql/connect/proto/commands_pb2.py
@@ -36,7 +36,7 @@ from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_rel
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
- b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto"\x86\x07\n\x07\x43ommand\x12]\n\x11register_function\x18\x01 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x10registerFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x [...]
+ b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto"\x86\x07\n\x07\x43ommand\x12]\n\x11register_function\x18\x01 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x10registerFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x [...]
)
@@ -487,75 +487,75 @@ if _descriptor._USE_C_DESCRIPTORS == False:
_COMMAND._serialized_start = 167
_COMMAND._serialized_end = 1069
_SQLCOMMAND._serialized_start = 1072
- _SQLCOMMAND._serialized_end = 1251
- _SQLCOMMAND_ARGSENTRY._serialized_start = 1161
- _SQLCOMMAND_ARGSENTRY._serialized_end = 1251
- _CREATEDATAFRAMEVIEWCOMMAND._serialized_start = 1254
- _CREATEDATAFRAMEVIEWCOMMAND._serialized_end = 1404
- _WRITEOPERATION._serialized_start = 1407
- _WRITEOPERATION._serialized_end = 2458
- _WRITEOPERATION_OPTIONSENTRY._serialized_start = 1882
- _WRITEOPERATION_OPTIONSENTRY._serialized_end = 1940
- _WRITEOPERATION_SAVETABLE._serialized_start = 1943
- _WRITEOPERATION_SAVETABLE._serialized_end = 2201
- _WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD._serialized_start = 2077
- _WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD._serialized_end = 2201
- _WRITEOPERATION_BUCKETBY._serialized_start = 2203
- _WRITEOPERATION_BUCKETBY._serialized_end = 2294
- _WRITEOPERATION_SAVEMODE._serialized_start = 2297
- _WRITEOPERATION_SAVEMODE._serialized_end = 2434
- _WRITEOPERATIONV2._serialized_start = 2461
- _WRITEOPERATIONV2._serialized_end = 3274
- _WRITEOPERATIONV2_OPTIONSENTRY._serialized_start = 1882
- _WRITEOPERATIONV2_OPTIONSENTRY._serialized_end = 1940
- _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_start = 3033
- _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_end = 3099
- _WRITEOPERATIONV2_MODE._serialized_start = 3102
- _WRITEOPERATIONV2_MODE._serialized_end = 3261
- _WRITESTREAMOPERATIONSTART._serialized_start = 3277
- _WRITESTREAMOPERATIONSTART._serialized_end = 3997
- _WRITESTREAMOPERATIONSTART_OPTIONSENTRY._serialized_start = 1882
- _WRITESTREAMOPERATIONSTART_OPTIONSENTRY._serialized_end = 1940
- _STREAMINGFOREACHWRITER._serialized_start = 4000
- _STREAMINGFOREACHWRITER._serialized_end = 4167
- _WRITESTREAMOPERATIONSTARTRESULT._serialized_start = 4169
- _WRITESTREAMOPERATIONSTARTRESULT._serialized_end = 4290
- _STREAMINGQUERYINSTANCEID._serialized_start = 4292
- _STREAMINGQUERYINSTANCEID._serialized_end = 4357
- _STREAMINGQUERYCOMMAND._serialized_start = 4360
- _STREAMINGQUERYCOMMAND._serialized_end = 4992
- _STREAMINGQUERYCOMMAND_EXPLAINCOMMAND._serialized_start = 4859
- _STREAMINGQUERYCOMMAND_EXPLAINCOMMAND._serialized_end = 4903
- _STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND._serialized_start = 4905
- _STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND._serialized_end = 4981
- _STREAMINGQUERYCOMMANDRESULT._serialized_start = 4995
- _STREAMINGQUERYCOMMANDRESULT._serialized_end = 6136
- _STREAMINGQUERYCOMMANDRESULT_STATUSRESULT._serialized_start = 5578
- _STREAMINGQUERYCOMMANDRESULT_STATUSRESULT._serialized_end = 5748
- _STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT._serialized_start = 5750
- _STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT._serialized_end = 5822
- _STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT._serialized_start = 5824
- _STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT._serialized_end = 5863
- _STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT._serialized_start = 5866
- _STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT._serialized_end = 6063
- _STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT._serialized_start = 6065
- _STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT._serialized_end = 6121
- _STREAMINGQUERYMANAGERCOMMAND._serialized_start = 6139
- _STREAMINGQUERYMANAGERCOMMAND._serialized_end = 6489
- _STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND._serialized_start = 6399
- _STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND._serialized_end = 6478
- _STREAMINGQUERYMANAGERCOMMANDRESULT._serialized_start = 6492
- _STREAMINGQUERYMANAGERCOMMANDRESULT._serialized_end = 7215
- _STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT._serialized_start = 6895
- _STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT._serialized_end = 7022
- _STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE._serialized_start = 7024
- _STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE._serialized_end = 7139
- _STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT._serialized_start = 7141
- _STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT._serialized_end = 7200
- _GETRESOURCESCOMMAND._serialized_start = 7217
- _GETRESOURCESCOMMAND._serialized_end = 7238
- _GETRESOURCESCOMMANDRESULT._serialized_start = 7241
- _GETRESOURCESCOMMANDRESULT._serialized_end = 7453
- _GETRESOURCESCOMMANDRESULT_RESOURCESENTRY._serialized_start = 7357
- _GETRESOURCESCOMMANDRESULT_RESOURCESENTRY._serialized_end = 7453
+ _SQLCOMMAND._serialized_end = 1313
+ _SQLCOMMAND_ARGSENTRY._serialized_start = 1223
+ _SQLCOMMAND_ARGSENTRY._serialized_end = 1313
+ _CREATEDATAFRAMEVIEWCOMMAND._serialized_start = 1316
+ _CREATEDATAFRAMEVIEWCOMMAND._serialized_end = 1466
+ _WRITEOPERATION._serialized_start = 1469
+ _WRITEOPERATION._serialized_end = 2520
+ _WRITEOPERATION_OPTIONSENTRY._serialized_start = 1944
+ _WRITEOPERATION_OPTIONSENTRY._serialized_end = 2002
+ _WRITEOPERATION_SAVETABLE._serialized_start = 2005
+ _WRITEOPERATION_SAVETABLE._serialized_end = 2263
+ _WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD._serialized_start = 2139
+ _WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD._serialized_end = 2263
+ _WRITEOPERATION_BUCKETBY._serialized_start = 2265
+ _WRITEOPERATION_BUCKETBY._serialized_end = 2356
+ _WRITEOPERATION_SAVEMODE._serialized_start = 2359
+ _WRITEOPERATION_SAVEMODE._serialized_end = 2496
+ _WRITEOPERATIONV2._serialized_start = 2523
+ _WRITEOPERATIONV2._serialized_end = 3336
+ _WRITEOPERATIONV2_OPTIONSENTRY._serialized_start = 1944
+ _WRITEOPERATIONV2_OPTIONSENTRY._serialized_end = 2002
+ _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_start = 3095
+ _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_end = 3161
+ _WRITEOPERATIONV2_MODE._serialized_start = 3164
+ _WRITEOPERATIONV2_MODE._serialized_end = 3323
+ _WRITESTREAMOPERATIONSTART._serialized_start = 3339
+ _WRITESTREAMOPERATIONSTART._serialized_end = 4059
+ _WRITESTREAMOPERATIONSTART_OPTIONSENTRY._serialized_start = 1944
+ _WRITESTREAMOPERATIONSTART_OPTIONSENTRY._serialized_end = 2002
+ _STREAMINGFOREACHWRITER._serialized_start = 4062
+ _STREAMINGFOREACHWRITER._serialized_end = 4229
+ _WRITESTREAMOPERATIONSTARTRESULT._serialized_start = 4231
+ _WRITESTREAMOPERATIONSTARTRESULT._serialized_end = 4352
+ _STREAMINGQUERYINSTANCEID._serialized_start = 4354
+ _STREAMINGQUERYINSTANCEID._serialized_end = 4419
+ _STREAMINGQUERYCOMMAND._serialized_start = 4422
+ _STREAMINGQUERYCOMMAND._serialized_end = 5054
+ _STREAMINGQUERYCOMMAND_EXPLAINCOMMAND._serialized_start = 4921
+ _STREAMINGQUERYCOMMAND_EXPLAINCOMMAND._serialized_end = 4965
+ _STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND._serialized_start = 4967
+ _STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND._serialized_end = 5043
+ _STREAMINGQUERYCOMMANDRESULT._serialized_start = 5057
+ _STREAMINGQUERYCOMMANDRESULT._serialized_end = 6198
+ _STREAMINGQUERYCOMMANDRESULT_STATUSRESULT._serialized_start = 5640
+ _STREAMINGQUERYCOMMANDRESULT_STATUSRESULT._serialized_end = 5810
+ _STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT._serialized_start = 5812
+ _STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT._serialized_end = 5884
+ _STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT._serialized_start = 5886
+ _STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT._serialized_end = 5925
+ _STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT._serialized_start = 5928
+ _STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT._serialized_end = 6125
+ _STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT._serialized_start = 6127
+ _STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT._serialized_end = 6183
+ _STREAMINGQUERYMANAGERCOMMAND._serialized_start = 6201
+ _STREAMINGQUERYMANAGERCOMMAND._serialized_end = 6551
+ _STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND._serialized_start = 6461
+ _STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND._serialized_end = 6540
+ _STREAMINGQUERYMANAGERCOMMANDRESULT._serialized_start = 6554
+ _STREAMINGQUERYMANAGERCOMMANDRESULT._serialized_end = 7277
+ _STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT._serialized_start = 6957
+ _STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT._serialized_end = 7084
+ _STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE._serialized_start = 7086
+ _STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE._serialized_end = 7201
+ _STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT._serialized_start = 7203
+ _STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT._serialized_end = 7262
+ _GETRESOURCESCOMMAND._serialized_start = 7279
+ _GETRESOURCESCOMMAND._serialized_end = 7300
+ _GETRESOURCESCOMMANDRESULT._serialized_start = 7303
+ _GETRESOURCESCOMMANDRESULT._serialized_end = 7515
+ _GETRESOURCESCOMMANDRESULT_RESOURCESENTRY._serialized_start = 7419
+ _GETRESOURCESCOMMANDRESULT_RESOURCESENTRY._serialized_end = 7515
# @@protoc_insertion_point(module_scope)
diff --git a/python/pyspark/sql/connect/proto/commands_pb2.pyi b/python/pyspark/sql/connect/proto/commands_pb2.pyi
index bbed1f0e252..3677af39fa1 100644
--- a/python/pyspark/sql/connect/proto/commands_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/commands_pb2.pyi
@@ -215,6 +215,7 @@ class SqlCommand(google.protobuf.message.Message):
SQL_FIELD_NUMBER: builtins.int
ARGS_FIELD_NUMBER: builtins.int
+ POS_ARGS_FIELD_NUMBER: builtins.int
sql: builtins.str
"""(Required) SQL Query."""
@property
@@ -224,6 +225,13 @@ class SqlCommand(google.protobuf.message.Message):
builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
]:
"""(Optional) A map of parameter names to literal expressions."""
+ @property
+ def pos_args(
+ self,
+ ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[
+ pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]:
+ """(Optional) A sequence of literal expressions for positional parameters in the SQL query text."""
def __init__(
self,
*,
@@ -232,9 +240,16 @@ class SqlCommand(google.protobuf.message.Message):
builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
]
| None = ...,
+ pos_args: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]
+ | None = ...,
) -> None: ...
def ClearField(
- self, field_name: typing_extensions.Literal["args", b"args", "sql", b"sql"]
+ self,
+ field_name: typing_extensions.Literal[
+ "args", b"args", "pos_args", b"pos_args", "sql", b"sql"
+ ],
) -> None: ...
global___SqlCommand = SqlCommand
diff --git a/python/pyspark/sql/connect/proto/relations_pb2.py b/python/pyspark/sql/connect/proto/relations_pb2.py
index 20e0a13c5e4..d175dedc1a2 100644
--- a/python/pyspark/sql/connect/proto/relations_pb2.py
+++ b/python/pyspark/sql/connect/proto/relations_pb2.py
@@ -36,7 +36,7 @@ from pyspark.sql.connect.proto import catalog_pb2 as spark_dot_connect_dot_catal
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
- b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x19spark/connect/types.proto\x1a\x1bspark/connect/catalog.proto"\xf3\x16\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66il [...]
+ b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x19spark/connect/types.proto\x1a\x1bspark/connect/catalog.proto"\xf3\x16\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66il [...]
)
@@ -778,125 +778,125 @@ if _descriptor._USE_C_DESCRIPTORS == False:
_RELATIONCOMMON._serialized_start = 3109
_RELATIONCOMMON._serialized_end = 3200
_SQL._serialized_start = 3203
- _SQL._serialized_end = 3372
- _SQL_ARGSENTRY._serialized_start = 3282
- _SQL_ARGSENTRY._serialized_end = 3372
- _READ._serialized_start = 3375
- _READ._serialized_end = 4038
- _READ_NAMEDTABLE._serialized_start = 3553
- _READ_NAMEDTABLE._serialized_end = 3745
- _READ_NAMEDTABLE_OPTIONSENTRY._serialized_start = 3687
- _READ_NAMEDTABLE_OPTIONSENTRY._serialized_end = 3745
- _READ_DATASOURCE._serialized_start = 3748
- _READ_DATASOURCE._serialized_end = 4025
- _READ_DATASOURCE_OPTIONSENTRY._serialized_start = 3687
- _READ_DATASOURCE_OPTIONSENTRY._serialized_end = 3745
- _PROJECT._serialized_start = 4040
- _PROJECT._serialized_end = 4157
- _FILTER._serialized_start = 4159
- _FILTER._serialized_end = 4271
- _JOIN._serialized_start = 4274
- _JOIN._serialized_end = 4745
- _JOIN_JOINTYPE._serialized_start = 4537
- _JOIN_JOINTYPE._serialized_end = 4745
- _SETOPERATION._serialized_start = 4748
- _SETOPERATION._serialized_end = 5227
- _SETOPERATION_SETOPTYPE._serialized_start = 5064
- _SETOPERATION_SETOPTYPE._serialized_end = 5178
- _LIMIT._serialized_start = 5229
- _LIMIT._serialized_end = 5305
- _OFFSET._serialized_start = 5307
- _OFFSET._serialized_end = 5386
- _TAIL._serialized_start = 5388
- _TAIL._serialized_end = 5463
- _AGGREGATE._serialized_start = 5466
- _AGGREGATE._serialized_end = 6048
- _AGGREGATE_PIVOT._serialized_start = 5805
- _AGGREGATE_PIVOT._serialized_end = 5916
- _AGGREGATE_GROUPTYPE._serialized_start = 5919
- _AGGREGATE_GROUPTYPE._serialized_end = 6048
- _SORT._serialized_start = 6051
- _SORT._serialized_end = 6211
- _DROP._serialized_start = 6214
- _DROP._serialized_end = 6355
- _DEDUPLICATE._serialized_start = 6358
- _DEDUPLICATE._serialized_end = 6598
- _LOCALRELATION._serialized_start = 6600
- _LOCALRELATION._serialized_end = 6689
- _CACHEDLOCALRELATION._serialized_start = 6691
- _CACHEDLOCALRELATION._serialized_end = 6786
- _SAMPLE._serialized_start = 6789
- _SAMPLE._serialized_end = 7062
- _RANGE._serialized_start = 7065
- _RANGE._serialized_end = 7210
- _SUBQUERYALIAS._serialized_start = 7212
- _SUBQUERYALIAS._serialized_end = 7326
- _REPARTITION._serialized_start = 7329
- _REPARTITION._serialized_end = 7471
- _SHOWSTRING._serialized_start = 7474
- _SHOWSTRING._serialized_end = 7616
- _HTMLSTRING._serialized_start = 7618
- _HTMLSTRING._serialized_end = 7732
- _STATSUMMARY._serialized_start = 7734
- _STATSUMMARY._serialized_end = 7826
- _STATDESCRIBE._serialized_start = 7828
- _STATDESCRIBE._serialized_end = 7909
- _STATCROSSTAB._serialized_start = 7911
- _STATCROSSTAB._serialized_end = 8012
- _STATCOV._serialized_start = 8014
- _STATCOV._serialized_end = 8110
- _STATCORR._serialized_start = 8113
- _STATCORR._serialized_end = 8250
- _STATAPPROXQUANTILE._serialized_start = 8253
- _STATAPPROXQUANTILE._serialized_end = 8417
- _STATFREQITEMS._serialized_start = 8419
- _STATFREQITEMS._serialized_end = 8544
- _STATSAMPLEBY._serialized_start = 8547
- _STATSAMPLEBY._serialized_end = 8856
- _STATSAMPLEBY_FRACTION._serialized_start = 8748
- _STATSAMPLEBY_FRACTION._serialized_end = 8847
- _NAFILL._serialized_start = 8859
- _NAFILL._serialized_end = 8993
- _NADROP._serialized_start = 8996
- _NADROP._serialized_end = 9130
- _NAREPLACE._serialized_start = 9133
- _NAREPLACE._serialized_end = 9429
- _NAREPLACE_REPLACEMENT._serialized_start = 9288
- _NAREPLACE_REPLACEMENT._serialized_end = 9429
- _TODF._serialized_start = 9431
- _TODF._serialized_end = 9519
- _WITHCOLUMNSRENAMED._serialized_start = 9522
- _WITHCOLUMNSRENAMED._serialized_end = 9761
- _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_start = 9694
- _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_end = 9761
- _WITHCOLUMNS._serialized_start = 9763
- _WITHCOLUMNS._serialized_end = 9882
- _WITHWATERMARK._serialized_start = 9885
- _WITHWATERMARK._serialized_end = 10019
- _HINT._serialized_start = 10022
- _HINT._serialized_end = 10154
- _UNPIVOT._serialized_start = 10157
- _UNPIVOT._serialized_end = 10484
- _UNPIVOT_VALUES._serialized_start = 10414
- _UNPIVOT_VALUES._serialized_end = 10473
- _TOSCHEMA._serialized_start = 10486
- _TOSCHEMA._serialized_end = 10592
- _REPARTITIONBYEXPRESSION._serialized_start = 10595
- _REPARTITIONBYEXPRESSION._serialized_end = 10798
- _MAPPARTITIONS._serialized_start = 10801
- _MAPPARTITIONS._serialized_end = 10982
- _GROUPMAP._serialized_start = 10985
- _GROUPMAP._serialized_end = 11620
- _COGROUPMAP._serialized_start = 11623
- _COGROUPMAP._serialized_end = 12149
- _APPLYINPANDASWITHSTATE._serialized_start = 12152
- _APPLYINPANDASWITHSTATE._serialized_end = 12509
- _COLLECTMETRICS._serialized_start = 12512
- _COLLECTMETRICS._serialized_end = 12648
- _PARSE._serialized_start = 12651
- _PARSE._serialized_end = 13039
- _PARSE_OPTIONSENTRY._serialized_start = 3687
- _PARSE_OPTIONSENTRY._serialized_end = 3745
- _PARSE_PARSEFORMAT._serialized_start = 12940
- _PARSE_PARSEFORMAT._serialized_end = 13028
+ _SQL._serialized_end = 3434
+ _SQL_ARGSENTRY._serialized_start = 3344
+ _SQL_ARGSENTRY._serialized_end = 3434
+ _READ._serialized_start = 3437
+ _READ._serialized_end = 4100
+ _READ_NAMEDTABLE._serialized_start = 3615
+ _READ_NAMEDTABLE._serialized_end = 3807
+ _READ_NAMEDTABLE_OPTIONSENTRY._serialized_start = 3749
+ _READ_NAMEDTABLE_OPTIONSENTRY._serialized_end = 3807
+ _READ_DATASOURCE._serialized_start = 3810
+ _READ_DATASOURCE._serialized_end = 4087
+ _READ_DATASOURCE_OPTIONSENTRY._serialized_start = 3749
+ _READ_DATASOURCE_OPTIONSENTRY._serialized_end = 3807
+ _PROJECT._serialized_start = 4102
+ _PROJECT._serialized_end = 4219
+ _FILTER._serialized_start = 4221
+ _FILTER._serialized_end = 4333
+ _JOIN._serialized_start = 4336
+ _JOIN._serialized_end = 4807
+ _JOIN_JOINTYPE._serialized_start = 4599
+ _JOIN_JOINTYPE._serialized_end = 4807
+ _SETOPERATION._serialized_start = 4810
+ _SETOPERATION._serialized_end = 5289
+ _SETOPERATION_SETOPTYPE._serialized_start = 5126
+ _SETOPERATION_SETOPTYPE._serialized_end = 5240
+ _LIMIT._serialized_start = 5291
+ _LIMIT._serialized_end = 5367
+ _OFFSET._serialized_start = 5369
+ _OFFSET._serialized_end = 5448
+ _TAIL._serialized_start = 5450
+ _TAIL._serialized_end = 5525
+ _AGGREGATE._serialized_start = 5528
+ _AGGREGATE._serialized_end = 6110
+ _AGGREGATE_PIVOT._serialized_start = 5867
+ _AGGREGATE_PIVOT._serialized_end = 5978
+ _AGGREGATE_GROUPTYPE._serialized_start = 5981
+ _AGGREGATE_GROUPTYPE._serialized_end = 6110
+ _SORT._serialized_start = 6113
+ _SORT._serialized_end = 6273
+ _DROP._serialized_start = 6276
+ _DROP._serialized_end = 6417
+ _DEDUPLICATE._serialized_start = 6420
+ _DEDUPLICATE._serialized_end = 6660
+ _LOCALRELATION._serialized_start = 6662
+ _LOCALRELATION._serialized_end = 6751
+ _CACHEDLOCALRELATION._serialized_start = 6753
+ _CACHEDLOCALRELATION._serialized_end = 6848
+ _SAMPLE._serialized_start = 6851
+ _SAMPLE._serialized_end = 7124
+ _RANGE._serialized_start = 7127
+ _RANGE._serialized_end = 7272
+ _SUBQUERYALIAS._serialized_start = 7274
+ _SUBQUERYALIAS._serialized_end = 7388
+ _REPARTITION._serialized_start = 7391
+ _REPARTITION._serialized_end = 7533
+ _SHOWSTRING._serialized_start = 7536
+ _SHOWSTRING._serialized_end = 7678
+ _HTMLSTRING._serialized_start = 7680
+ _HTMLSTRING._serialized_end = 7794
+ _STATSUMMARY._serialized_start = 7796
+ _STATSUMMARY._serialized_end = 7888
+ _STATDESCRIBE._serialized_start = 7890
+ _STATDESCRIBE._serialized_end = 7971
+ _STATCROSSTAB._serialized_start = 7973
+ _STATCROSSTAB._serialized_end = 8074
+ _STATCOV._serialized_start = 8076
+ _STATCOV._serialized_end = 8172
+ _STATCORR._serialized_start = 8175
+ _STATCORR._serialized_end = 8312
+ _STATAPPROXQUANTILE._serialized_start = 8315
+ _STATAPPROXQUANTILE._serialized_end = 8479
+ _STATFREQITEMS._serialized_start = 8481
+ _STATFREQITEMS._serialized_end = 8606
+ _STATSAMPLEBY._serialized_start = 8609
+ _STATSAMPLEBY._serialized_end = 8918
+ _STATSAMPLEBY_FRACTION._serialized_start = 8810
+ _STATSAMPLEBY_FRACTION._serialized_end = 8909
+ _NAFILL._serialized_start = 8921
+ _NAFILL._serialized_end = 9055
+ _NADROP._serialized_start = 9058
+ _NADROP._serialized_end = 9192
+ _NAREPLACE._serialized_start = 9195
+ _NAREPLACE._serialized_end = 9491
+ _NAREPLACE_REPLACEMENT._serialized_start = 9350
+ _NAREPLACE_REPLACEMENT._serialized_end = 9491
+ _TODF._serialized_start = 9493
+ _TODF._serialized_end = 9581
+ _WITHCOLUMNSRENAMED._serialized_start = 9584
+ _WITHCOLUMNSRENAMED._serialized_end = 9823
+ _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_start = 9756
+ _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_end = 9823
+ _WITHCOLUMNS._serialized_start = 9825
+ _WITHCOLUMNS._serialized_end = 9944
+ _WITHWATERMARK._serialized_start = 9947
+ _WITHWATERMARK._serialized_end = 10081
+ _HINT._serialized_start = 10084
+ _HINT._serialized_end = 10216
+ _UNPIVOT._serialized_start = 10219
+ _UNPIVOT._serialized_end = 10546
+ _UNPIVOT_VALUES._serialized_start = 10476
+ _UNPIVOT_VALUES._serialized_end = 10535
+ _TOSCHEMA._serialized_start = 10548
+ _TOSCHEMA._serialized_end = 10654
+ _REPARTITIONBYEXPRESSION._serialized_start = 10657
+ _REPARTITIONBYEXPRESSION._serialized_end = 10860
+ _MAPPARTITIONS._serialized_start = 10863
+ _MAPPARTITIONS._serialized_end = 11044
+ _GROUPMAP._serialized_start = 11047
+ _GROUPMAP._serialized_end = 11682
+ _COGROUPMAP._serialized_start = 11685
+ _COGROUPMAP._serialized_end = 12211
+ _APPLYINPANDASWITHSTATE._serialized_start = 12214
+ _APPLYINPANDASWITHSTATE._serialized_end = 12571
+ _COLLECTMETRICS._serialized_start = 12574
+ _COLLECTMETRICS._serialized_end = 12710
+ _PARSE._serialized_start = 12713
+ _PARSE._serialized_end = 13101
+ _PARSE_OPTIONSENTRY._serialized_start = 3749
+ _PARSE_OPTIONSENTRY._serialized_end = 3807
+ _PARSE_PARSEFORMAT._serialized_start = 13002
+ _PARSE_PARSEFORMAT._serialized_end = 13090
# @@protoc_insertion_point(module_scope)
diff --git a/python/pyspark/sql/connect/proto/relations_pb2.pyi b/python/pyspark/sql/connect/proto/relations_pb2.pyi
index bd6460519a4..28609b41713 100644
--- a/python/pyspark/sql/connect/proto/relations_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/relations_pb2.pyi
@@ -613,6 +613,7 @@ class SQL(google.protobuf.message.Message):
QUERY_FIELD_NUMBER: builtins.int
ARGS_FIELD_NUMBER: builtins.int
+ POS_ARGS_FIELD_NUMBER: builtins.int
query: builtins.str
"""(Required) The SQL query."""
@property
@@ -622,6 +623,13 @@ class SQL(google.protobuf.message.Message):
builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
]:
"""(Optional) A map of parameter names to literal expressions."""
+ @property
+ def pos_args(
+ self,
+ ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[
+ pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]:
+ """(Optional) A sequence of literal expressions for positional parameters in the SQL query text."""
def __init__(
self,
*,
@@ -630,9 +638,16 @@ class SQL(google.protobuf.message.Message):
builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
]
| None = ...,
+ pos_args: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]
+ | None = ...,
) -> None: ...
def ClearField(
- self, field_name: typing_extensions.Literal["args", b"args", "query", b"query"]
+ self,
+ field_name: typing_extensions.Literal[
+ "args", b"args", "pos_args", b"pos_args", "query", b"query"
+ ],
) -> None: ...
global___SQL = SQL
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org