You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2023/03/27 05:54:47 UTC
[spark] branch branch-3.4 updated: [SPARK-42924][SQL][CONNECT][PYTHON] Clarify the comment of parameterized SQL args
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new dde9de60483 [SPARK-42924][SQL][CONNECT][PYTHON] Clarify the comment of parameterized SQL args
dde9de60483 is described below
commit dde9de60483ebb0b57013387dcfb8d5075cb6d0c
Author: Max Gekk <ma...@gmail.com>
AuthorDate: Mon Mar 27 08:54:18 2023 +0300
[SPARK-42924][SQL][CONNECT][PYTHON] Clarify the comment of parameterized SQL args
### What changes were proposed in this pull request?
In the PR, I propose to clarify the comment of `args` in parameterized `sql()`.
### Why are the changes needed?
To make the comment more clear and highlight that input strings are parsed (not evaluated), and considered as SQL literal expressions. Also while parsing the fragments w/ SQL comments in the string values are skipped.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
By checking coding style:
```
$ ./dev/lint-python
$ ./dev/scalastyle
```
Closes #40508 from MaxGekk/parameterized-sql-doc.
Authored-by: Max Gekk <ma...@gmail.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
(cherry picked from commit c55c7ea6fc92c3733543d5f3d99eb00921cbe564)
Signed-off-by: Max Gekk <ma...@gmail.com>
---
.../jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala | 10 ++++++++--
.../common/src/main/protobuf/spark/connect/commands.proto | 5 ++++-
.../common/src/main/protobuf/spark/connect/relations.proto | 5 ++++-
python/pyspark/pandas/sql_formatter.py | 6 ++++--
python/pyspark/sql/connect/proto/commands_pb2.pyi | 6 +++++-
python/pyspark/sql/connect/proto/relations_pb2.pyi | 6 +++++-
python/pyspark/sql/session.py | 6 ++++--
.../src/main/scala/org/apache/spark/sql/SparkSession.scala | 10 ++++++++--
8 files changed, 42 insertions(+), 12 deletions(-)
diff --git a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala
index f1e82507393..548545b969d 100644
--- a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -213,7 +213,10 @@ class SparkSession private[sql] (
* @param sqlText
* A SQL statement with named parameters to execute.
* @param args
- * A map of parameter names to literal values.
+ * A map of parameter names to string values that are parsed as SQL literal expressions. For
+ * example, map keys: "rank", "name", "birthdate"; map values: "1", "'Steven'",
+ * "DATE'2023-03-21'". The fragments of string values belonged to SQL comments are skipped
+ * while parsing.
*
* @since 3.4.0
*/
@@ -229,7 +232,10 @@ class SparkSession private[sql] (
* @param sqlText
* A SQL statement with named parameters to execute.
* @param args
- * A map of parameter names to literal values.
+ * A map of parameter names to string values that are parsed as SQL literal expressions. For
+ * example, map keys: "rank", "name", "birthdate"; map values: "1", "'Steven'",
+ * "DATE'2023-03-21'". The fragments of string values belonged to SQL comments are skipped
+ * while parsing.
*
* @since 3.4.0
*/
diff --git a/connector/connect/common/src/main/protobuf/spark/connect/commands.proto b/connector/connect/common/src/main/protobuf/spark/connect/commands.proto
index 3ffbe83bded..604421fdd4f 100644
--- a/connector/connect/common/src/main/protobuf/spark/connect/commands.proto
+++ b/connector/connect/common/src/main/protobuf/spark/connect/commands.proto
@@ -53,7 +53,10 @@ message SqlCommand {
// (Required) SQL Query.
string sql = 1;
- // (Optional) A map of parameter names to literal values.
+ // (Optional) A map of parameter names to string values that are parsed as
+ // SQL literal expressions. For example, map keys: "rank", "name", "birthdate";
+ // map values: "1", "'Steven'", "DATE'2023-03-21'". The fragments of string values
+ // belonged to SQL comments are skipped while parsing.
map<string, string> args = 2;
}
diff --git a/connector/connect/common/src/main/protobuf/spark/connect/relations.proto b/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
index 9dec94411a9..976bd68e7fe 100644
--- a/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
+++ b/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
@@ -108,7 +108,10 @@ message SQL {
// (Required) The SQL query.
string query = 1;
- // (Optional) A map of parameter names to literal values.
+ // (Optional) A map of parameter names to string values that are parsed as
+ // SQL literal expressions. For example, map keys: "rank", "name", "birthdate";
+ // map values: "1", "'Steven'", "DATE'2023-03-21'". The fragments of string values
+ // belonged to SQL comments are skipped while parsing.
map<string, string> args = 2;
}
diff --git a/python/pyspark/pandas/sql_formatter.py b/python/pyspark/pandas/sql_formatter.py
index 9103366c192..f87dd3ff29f 100644
--- a/python/pyspark/pandas/sql_formatter.py
+++ b/python/pyspark/pandas/sql_formatter.py
@@ -103,8 +103,10 @@ def sql(
Also note that the index name(s) should be matched to the existing name.
args : dict
- A dictionary of named parameters that begin from the `:` marker and
- their SQL literals for substituting.
+ A dictionary of parameter names to string values that are parsed as SQL literal
+ expressions. For example, dict keys: "rank", "name", "birthdate"; dict values:
+ "1", "'Steven'", "DATE'2023-03-21'". The fragments of string values belonged to SQL
+ comments are skipped while parsing.
.. versionadded:: 3.4.0
diff --git a/python/pyspark/sql/connect/proto/commands_pb2.pyi b/python/pyspark/sql/connect/proto/commands_pb2.pyi
index d2bfaf9ed89..b01745e888f 100644
--- a/python/pyspark/sql/connect/proto/commands_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/commands_pb2.pyi
@@ -178,7 +178,11 @@ class SqlCommand(google.protobuf.message.Message):
"""(Required) SQL Query."""
@property
def args(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
- """(Optional) A map of parameter names to literal values."""
+ """(Optional) A map of parameter names to string values that are parsed as
+ SQL literal expressions. For example, map keys: "rank", "name", "birthdate";
+ map values: "1", "'Steven'", "DATE'2023-03-21'". The fragments of string values
+ belonged to SQL comments are skipped while parsing.
+ """
def __init__(
self,
*,
diff --git a/python/pyspark/sql/connect/proto/relations_pb2.pyi b/python/pyspark/sql/connect/proto/relations_pb2.pyi
index 1afedbaf9a2..f287a740346 100644
--- a/python/pyspark/sql/connect/proto/relations_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/relations_pb2.pyi
@@ -577,7 +577,11 @@ class SQL(google.protobuf.message.Message):
"""(Required) The SQL query."""
@property
def args(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
- """(Optional) A map of parameter names to literal values."""
+ """(Optional) A map of parameter names to string values that are parsed as
+ SQL literal expressions. For example, map keys: "rank", "name", "birthdate";
+ map values: "1", "'Steven'", "DATE'2023-03-21'". The fragments of string values
+ belonged to SQL comments are skipped while parsing.
+ """
def __init__(
self,
*,
diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py
index da234abf927..8818092e5b5 100644
--- a/python/pyspark/sql/session.py
+++ b/python/pyspark/sql/session.py
@@ -1336,8 +1336,10 @@ class SparkSession(SparkConversionMixin):
sqlQuery : str
SQL query string.
args : dict
- A dictionary of named parameters that begin from the `:` marker and
- their SQL literals for substituting.
+ A dictionary of parameter names to string values that are parsed as SQL literal
+ expressions. For example, dict keys: "rank", "name", "birthdate"; dict values:
+ "1", "'Steven'", "DATE'2023-03-21'". The fragments of string values belonged to
+ SQL comments are skipped while parsing.
.. versionadded:: 3.4.0
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 066e609a6d3..f9411a10fa6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -614,7 +614,10 @@ class SparkSession private(
* This API eagerly runs DDL/DML commands, but not for SELECT queries.
*
* @param sqlText A SQL statement with named parameters to execute.
- * @param args A map of parameter names to literal values.
+ * @param args A map of parameter names to string values that are parsed as
+ * SQL literal expressions. For example, map keys: "rank", "name", "birthdate";
+ * map values: "1", "'Steven'", "DATE'2023-03-21'". The fragments of string values
+ * belonged to SQL comments are skipped while parsing.
*
* @since 3.4.0
*/
@@ -639,7 +642,10 @@ class SparkSession private(
* This API eagerly runs DDL/DML commands, but not for SELECT queries.
*
* @param sqlText A SQL statement with named parameters to execute.
- * @param args A map of parameter names to literal values.
+ * @param args A map of parameter names to string values that are parsed as
+ * SQL literal expressions. For example, map keys: "rank", "name", "birthdate";
+ * map values: "1", "'Steven'", "DATE'2023-03-21'". The fragments of string values
+ * belonged to SQL comments are skipped while parsing.
*
* @since 3.4.0
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org