You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2025/01/16 02:44:55 UTC
(spark) branch branch-4.0 updated: [SPARK-50804][SQL] to_protobuf() should not throw MatchError
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new d2530db937bb [SPARK-50804][SQL] to_protobuf() should not throw MatchError
d2530db937bb is described below
commit d2530db937bbe65f8c92bb1b958f5983998dc8d7
Author: HyukjinKwon <gu...@apache.org>
AuthorDate: Thu Jan 16 11:44:31 2025 +0900
[SPARK-50804][SQL] to_protobuf() should not throw MatchError
### What changes were proposed in this pull request?
Right now, if `to_protobuf()` is called with an argument that is not StructType, `
scala.MatchError` is thrown. We should categorize this error.
### Why are the changes needed?
`scala.MatchError` is confusing to users and doesn't include enough information for debugging.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Add a new unit test.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #49473
Closes #49520 from HyukjinKwon/SPARK-50804.
Lead-authored-by: HyukjinKwon <gu...@apache.org>
Co-authored-by: Siying Dong <do...@gmail.com>
Co-authored-by: Hyukjin Kwon <gu...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
(cherry picked from commit c3671a592ddd8a4e325cb63a5dfc937f7080da3f)
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../src/main/resources/error/error-conditions.json | 5 +++
.../apache/spark/sql/PlanGenerationTestSuite.scala | 20 +++++++---
.../spark/sql/protobuf/ProtobufSerializer.scala | 3 ++
.../sql/protobuf/ProtobufFunctionsSuite.scala | 27 ++++++++++++++
.../expressions/toFromProtobufSqlFunctions.scala | 24 +++++++++---
.../to_protobuf_messageClassName.explain | 5 ++-
..._protobuf_messageClassName_descFilePath.explain | 5 ++-
...f_messageClassName_descFilePath_options.explain | 5 ++-
.../to_protobuf_messageClassName_options.explain | 5 ++-
.../queries/to_protobuf_messageClassName.json | 41 ++++++++++++++++++---
.../queries/to_protobuf_messageClassName.proto.bin | Bin 125 -> 461 bytes
.../to_protobuf_messageClassName_descFilePath.json | 41 ++++++++++++++++++---
...rotobuf_messageClassName_descFilePath.proto.bin | Bin 361 -> 696 bytes
...obuf_messageClassName_descFilePath_options.json | 41 ++++++++++++++++++---
...messageClassName_descFilePath_options.proto.bin | Bin 411 -> 746 bytes
.../to_protobuf_messageClassName_options.json | 41 ++++++++++++++++++---
.../to_protobuf_messageClassName_options.proto.bin | Bin 176 -> 511 bytes
17 files changed, 224 insertions(+), 39 deletions(-)
diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json
index ed4bee9b89f1..3b4e49999569 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -988,6 +988,11 @@
"all arguments of the function <funcName> must be strings."
]
},
+ "NON_STRUCT_TYPE" : {
+ "message" : [
+ "the input <inputName> should be a struct expression; however, got <inputType>."
+ ]
+ },
"NULL_TYPE" : {
"message" : [
"Null typed values cannot be used as arguments of <functionName>."
diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
index 1b973ac8ea34..067a7380561d 100644
--- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
+++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
@@ -3492,27 +3492,35 @@ class PlanGenerationTestSuite
}
test("to_protobuf messageClassName") {
- binary.select(pbFn.to_protobuf(fn.col("bytes"), classOf[StorageLevel].getName))
+ val df = binary.select(
+ pbFn.from_protobuf(fn.col("bytes"), "StorageLevel", testDescFilePath).alias("col"))
+ df.select(pbFn.to_protobuf(fn.col("col"), classOf[StorageLevel].getName))
}
test("to_protobuf messageClassName options") {
- binary.select(
+ val df = binary.select(
+ pbFn.from_protobuf(fn.col("bytes"), "StorageLevel", testDescFilePath).alias("col"))
+ df.select(
pbFn.to_protobuf(
- fn.col("bytes"),
+ fn.col("col"),
classOf[StorageLevel].getName,
Map("recursive.fields.max.depth" -> "2").asJava))
}
test("to_protobuf messageClassName descFilePath options") {
- binary.select(
+ val df = binary.select(
+ pbFn.from_protobuf(fn.col("bytes"), "StorageLevel", testDescFilePath).alias("col"))
+ df.select(
pbFn.to_protobuf(
- fn.col("bytes"),
+ fn.col("col"),
"StorageLevel",
testDescFilePath,
Map("recursive.fields.max.depth" -> "2").asJava))
}
test("to_protobuf messageClassName descFilePath") {
- binary.select(pbFn.to_protobuf(fn.col("bytes"), "StorageLevel", testDescFilePath))
+ val df = binary.select(
+ pbFn.from_protobuf(fn.col("bytes"), "StorageLevel", testDescFilePath).alias("col"))
+ df.select(pbFn.to_protobuf(fn.col("col"), "StorageLevel", testDescFilePath))
}
}
diff --git a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/ProtobufSerializer.scala b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/ProtobufSerializer.scala
index 1c64e70755d5..65e8cce0d056 100644
--- a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/ProtobufSerializer.scala
+++ b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/ProtobufSerializer.scala
@@ -47,6 +47,9 @@ private[sql] class ProtobufSerializer(
}
private val converter: Any => Any = {
+ assert(
+ rootCatalystType.isInstanceOf[StructType],
+ "ProtobufSerializer's root catalyst type must be a struct type")
val baseConverter =
try {
rootCatalystType match {
diff --git a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
index 44a8339ac1f0..c90ca239c1d4 100644
--- a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
+++ b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
@@ -1721,6 +1721,33 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Prot
}
}
+ test("non-struct SQL type") {
+ val dfWithInt = spark
+ .range(1)
+ .select(
+ lit(9999).as("int_col")
+ )
+
+ val parseError = intercept[AnalysisException] {
+ dfWithInt.select(
+ to_protobuf_wrapper($"int_col", "SimpleMessageEnum", Some(testFileDesc))).collect()
+ }
+ val descMsg = testFileDesc.map("%02X".format(_)).mkString("")
+ checkError(
+ exception = parseError,
+ condition = "DATATYPE_MISMATCH.NON_STRUCT_TYPE",
+ parameters = Map(
+ "inputName" -> "data",
+ "inputType" -> "\"INT\"",
+ "sqlExpr" ->
+ s"""\"to_protobuf(int_col, SimpleMessageEnum, X'$descMsg', NULL)\""""
+ ),
+ queryContext = Array(ExpectedContext(
+ fragment = "fn",
+ callSitePattern = ".*"))
+ )
+ }
+
test("test unsigned integer types") {
// Test that we correctly handle unsigned integer parsing.
// We're using Integer/Long's `MIN_VALUE` as it has a 1 in the sign bit.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromProtobufSqlFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromProtobufSqlFunctions.scala
index 96bcf49dbd09..d6167d925bd2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromProtobufSqlFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromProtobufSqlFunctions.scala
@@ -20,8 +20,9 @@ package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.util.ArrayBasedMapData
+import org.apache.spark.sql.errors.DataTypeErrors.toSQLType
import org.apache.spark.sql.errors.QueryCompilationErrors
-import org.apache.spark.sql.types.{BinaryType, MapType, NullType, StringType}
+import org.apache.spark.sql.types.{BinaryType, MapType, NullType, StringType, StructType}
import org.apache.spark.sql.util.ProtobufUtils
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.Utils
@@ -238,6 +239,18 @@ case class ToProtobuf(
}
override def checkInputDataTypes(): TypeCheckResult = {
+ val colTypeCheck = first.dataType match {
+ case _: StructType => None
+ case _ =>
+ Some(
+ TypeCheckResult.DataTypeMismatch(
+ errorSubClass = "NON_STRUCT_TYPE",
+ messageParameters = Map(
+ "inputName" -> "data",
+ "inputType" -> toSQLType(first.dataType)))
+ )
+ }
+
val messageNameCheck = messageName.dataType match {
case _: StringType if messageName.foldable => None
case _ =>
@@ -262,10 +275,11 @@ case class ToProtobuf(
"strings to strings containing the options to use for converting the value to " +
"Protobuf format"))
}
-
- messageNameCheck.getOrElse(
- descFilePathCheck.getOrElse(
- optionsCheck.getOrElse(TypeCheckResult.TypeCheckSuccess)
+ colTypeCheck.getOrElse(
+ messageNameCheck.getOrElse(
+ descFilePathCheck.getOrElse(
+ optionsCheck.getOrElse(TypeCheckResult.TypeCheckSuccess)
+ )
)
)
}
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName.explain
index 3533406c0bf0..33f3a8705880 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName.explain
@@ -1,2 +1,3 @@
-Project [to_protobuf(bytes#0, org.apache.spark.connect.proto.StorageLevel, None) AS to_protobuf(bytes, org.apache.spark.connect.proto.StorageLevel, NULL, NULL)#0]
-+- LocalRelation <empty>, [id#0L, bytes#0]
+Project [to_protobuf(col#0, org.apache.spark.connect.proto.StorageLevel, None) AS to_protobuf(col, org.apache.spark.connect.proto.StorageLevel, NULL, NULL)#0]
++- Project [from_protobuf(bytes#0, StorageLevel, Some([B)) AS col#0]
+ +- LocalRelation <empty>, [id#0L, bytes#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath.explain
index f6a33a20a5dc..77bcdc60f18c 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath.explain
@@ -1,2 +1,3 @@
-Project [to_protobuf(bytes#0, StorageLevel, Some([B)) AS to_protobuf(bytes, StorageLevel, X'0AFC010A0C636F6D6D6F6E2E70726F746F120D737061726B2E636F6E6E65637422B0010A0C53746F726167654C6576656C12190A087573655F6469736B18012001280852077573654469736B121D0A0A7573655F6D656D6F727918022001280852097573654D656D6F727912200A0C7573655F6F66665F68656170180320012808520A7573654F66664865617012220A0C646573657269616C697A6564180420012808520C646573657269616C697A656412200A0B7265706C69636174696F6E180520012805520B [...]
-+- LocalRelation <empty>, [id#0L, bytes#0]
+Project [to_protobuf(col#0, StorageLevel, Some([B)) AS to_protobuf(col, StorageLevel, X'0AFC010A0C636F6D6D6F6E2E70726F746F120D737061726B2E636F6E6E65637422B0010A0C53746F726167654C6576656C12190A087573655F6469736B18012001280852077573654469736B121D0A0A7573655F6D656D6F727918022001280852097573654D656D6F727912200A0C7573655F6F66665F68656170180320012808520A7573654F66664865617012220A0C646573657269616C697A6564180420012808520C646573657269616C697A656412200A0B7265706C69636174696F6E180520012805520B7265 [...]
++- Project [from_protobuf(bytes#0, StorageLevel, Some([B)) AS col#0]
+ +- LocalRelation <empty>, [id#0L, bytes#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath_options.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath_options.explain
index 393529a15670..a01215623d7d 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath_options.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath_options.explain
@@ -1,2 +1,3 @@
-Project [to_protobuf(bytes#0, StorageLevel, Some([B), (recursive.fields.max.depth,2)) AS to_protobuf(bytes, StorageLevel, X'0AFC010A0C636F6D6D6F6E2E70726F746F120D737061726B2E636F6E6E65637422B0010A0C53746F726167654C6576656C12190A087573655F6469736B18012001280852077573654469736B121D0A0A7573655F6D656D6F727918022001280852097573654D656D6F727912200A0C7573655F6F66665F68656170180320012808520A7573654F66664865617012220A0C646573657269616C697A6564180420012808520C646573657269616C697A656412200A0B726570 [...]
-+- LocalRelation <empty>, [id#0L, bytes#0]
+Project [to_protobuf(col#0, StorageLevel, Some([B), (recursive.fields.max.depth,2)) AS to_protobuf(col, StorageLevel, X'0AFC010A0C636F6D6D6F6E2E70726F746F120D737061726B2E636F6E6E65637422B0010A0C53746F726167654C6576656C12190A087573655F6469736B18012001280852077573654469736B121D0A0A7573655F6D656D6F727918022001280852097573654D656D6F727912200A0C7573655F6F66665F68656170180320012808520A7573654F66664865617012220A0C646573657269616C697A6564180420012808520C646573657269616C697A656412200A0B7265706C69 [...]
++- Project [from_protobuf(bytes#0, StorageLevel, Some([B)) AS col#0]
+ +- LocalRelation <empty>, [id#0L, bytes#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_options.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_options.explain
index e0c7e1625fe5..3c18f981cb8e 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_options.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_options.explain
@@ -1,2 +1,3 @@
-Project [to_protobuf(bytes#0, org.apache.spark.connect.proto.StorageLevel, None, (recursive.fields.max.depth,2)) AS to_protobuf(bytes, org.apache.spark.connect.proto.StorageLevel, NULL, map(recursive.fields.max.depth, 2))#0]
-+- LocalRelation <empty>, [id#0L, bytes#0]
+Project [to_protobuf(col#0, org.apache.spark.connect.proto.StorageLevel, None, (recursive.fields.max.depth,2)) AS to_protobuf(col, org.apache.spark.connect.proto.StorageLevel, NULL, map(recursive.fields.max.depth, 2))#0]
++- Project [from_protobuf(bytes#0, StorageLevel, Some([B)) AS col#0]
+ +- LocalRelation <empty>, [id#0L, bytes#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.json b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.json
index e71bddc0b19a..5eb4ba27d97c 100644
--- a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.json
+++ b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.json
@@ -1,14 +1,45 @@
{
"common": {
- "planId": "1"
+ "planId": "2"
},
"project": {
"input": {
"common": {
- "planId": "0"
+ "planId": "1"
},
- "localRelation": {
- "schema": "struct\u003cid:bigint,bytes:binary\u003e"
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema": "struct\u003cid:bigint,bytes:binary\u003e"
+ }
+ },
+ "expressions": [{
+ "alias": {
+ "expr": {
+ "unresolvedFunction": {
+ "functionName": "from_protobuf",
+ "arguments": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "bytes"
+ }
+ }, {
+ "literal": {
+ "string": "StorageLevel"
+ }
+ }, {
+ "literal": {
+ "binary": "CvwBCgxjb21tb24ucHJvdG8SDXNwYXJrLmNvbm5lY3QisAEKDFN0b3JhZ2VMZXZlbBIZCgh1c2VfZGlzaxgBIAEoCFIHdXNlRGlzaxIdCgp1c2VfbWVtb3J5GAIgASgIUgl1c2VNZW1vcnkSIAoMdXNlX29mZl9oZWFwGAMgASgIUgp1c2VPZmZIZWFwEiIKDGRlc2VyaWFsaXplZBgEIAEoCFIMZGVzZXJpYWxpemVkEiAKC3JlcGxpY2F0aW9uGAUgASgFUgtyZXBsaWNhdGlvbkIiCh5vcmcuYXBhY2hlLnNwYXJrLmNvbm5lY3QucHJvdG9QAWIGcHJvdG8z"
+ }
+ }],
+ "isInternal": false
+ }
+ },
+ "name": ["col"]
+ }
+ }]
}
},
"expressions": [{
@@ -16,7 +47,7 @@
"functionName": "to_protobuf",
"arguments": [{
"unresolvedAttribute": {
- "unparsedIdentifier": "bytes"
+ "unparsedIdentifier": "col"
}
}, {
"literal": {
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.proto.bin
index f49e6d227ddc..eac490238cf8 100644
Binary files a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.proto.bin differ
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.json b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.json
index c6ccee6f35c3..f85a2e708ef3 100644
--- a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.json
+++ b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.json
@@ -1,14 +1,45 @@
{
"common": {
- "planId": "1"
+ "planId": "2"
},
"project": {
"input": {
"common": {
- "planId": "0"
+ "planId": "1"
},
- "localRelation": {
- "schema": "struct\u003cid:bigint,bytes:binary\u003e"
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema": "struct\u003cid:bigint,bytes:binary\u003e"
+ }
+ },
+ "expressions": [{
+ "alias": {
+ "expr": {
+ "unresolvedFunction": {
+ "functionName": "from_protobuf",
+ "arguments": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "bytes"
+ }
+ }, {
+ "literal": {
+ "string": "StorageLevel"
+ }
+ }, {
+ "literal": {
+ "binary": "CvwBCgxjb21tb24ucHJvdG8SDXNwYXJrLmNvbm5lY3QisAEKDFN0b3JhZ2VMZXZlbBIZCgh1c2VfZGlzaxgBIAEoCFIHdXNlRGlzaxIdCgp1c2VfbWVtb3J5GAIgASgIUgl1c2VNZW1vcnkSIAoMdXNlX29mZl9oZWFwGAMgASgIUgp1c2VPZmZIZWFwEiIKDGRlc2VyaWFsaXplZBgEIAEoCFIMZGVzZXJpYWxpemVkEiAKC3JlcGxpY2F0aW9uGAUgASgFUgtyZXBsaWNhdGlvbkIiCh5vcmcuYXBhY2hlLnNwYXJrLmNvbm5lY3QucHJvdG9QAWIGcHJvdG8z"
+ }
+ }],
+ "isInternal": false
+ }
+ },
+ "name": ["col"]
+ }
+ }]
}
},
"expressions": [{
@@ -16,7 +47,7 @@
"functionName": "to_protobuf",
"arguments": [{
"unresolvedAttribute": {
- "unparsedIdentifier": "bytes"
+ "unparsedIdentifier": "col"
}
}, {
"literal": {
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.proto.bin
index c30bc963ce0e..5771484d061c 100644
Binary files a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.proto.bin differ
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.json b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.json
index bc676b7aa5b1..f2ed7c8f057c 100644
--- a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.json
+++ b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.json
@@ -1,14 +1,45 @@
{
"common": {
- "planId": "1"
+ "planId": "2"
},
"project": {
"input": {
"common": {
- "planId": "0"
+ "planId": "1"
},
- "localRelation": {
- "schema": "struct\u003cid:bigint,bytes:binary\u003e"
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema": "struct\u003cid:bigint,bytes:binary\u003e"
+ }
+ },
+ "expressions": [{
+ "alias": {
+ "expr": {
+ "unresolvedFunction": {
+ "functionName": "from_protobuf",
+ "arguments": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "bytes"
+ }
+ }, {
+ "literal": {
+ "string": "StorageLevel"
+ }
+ }, {
+ "literal": {
+ "binary": "CvwBCgxjb21tb24ucHJvdG8SDXNwYXJrLmNvbm5lY3QisAEKDFN0b3JhZ2VMZXZlbBIZCgh1c2VfZGlzaxgBIAEoCFIHdXNlRGlzaxIdCgp1c2VfbWVtb3J5GAIgASgIUgl1c2VNZW1vcnkSIAoMdXNlX29mZl9oZWFwGAMgASgIUgp1c2VPZmZIZWFwEiIKDGRlc2VyaWFsaXplZBgEIAEoCFIMZGVzZXJpYWxpemVkEiAKC3JlcGxpY2F0aW9uGAUgASgFUgtyZXBsaWNhdGlvbkIiCh5vcmcuYXBhY2hlLnNwYXJrLmNvbm5lY3QucHJvdG9QAWIGcHJvdG8z"
+ }
+ }],
+ "isInternal": false
+ }
+ },
+ "name": ["col"]
+ }
+ }]
}
},
"expressions": [{
@@ -16,7 +47,7 @@
"functionName": "to_protobuf",
"arguments": [{
"unresolvedAttribute": {
- "unparsedIdentifier": "bytes"
+ "unparsedIdentifier": "col"
}
}, {
"literal": {
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.proto.bin
index 971d6b358711..893efcc1c018 100644
Binary files a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.proto.bin differ
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.json b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.json
index fa3d57a251cb..f8cd4595fd1e 100644
--- a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.json
+++ b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.json
@@ -1,14 +1,45 @@
{
"common": {
- "planId": "1"
+ "planId": "2"
},
"project": {
"input": {
"common": {
- "planId": "0"
+ "planId": "1"
},
- "localRelation": {
- "schema": "struct\u003cid:bigint,bytes:binary\u003e"
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema": "struct\u003cid:bigint,bytes:binary\u003e"
+ }
+ },
+ "expressions": [{
+ "alias": {
+ "expr": {
+ "unresolvedFunction": {
+ "functionName": "from_protobuf",
+ "arguments": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "bytes"
+ }
+ }, {
+ "literal": {
+ "string": "StorageLevel"
+ }
+ }, {
+ "literal": {
+ "binary": "CvwBCgxjb21tb24ucHJvdG8SDXNwYXJrLmNvbm5lY3QisAEKDFN0b3JhZ2VMZXZlbBIZCgh1c2VfZGlzaxgBIAEoCFIHdXNlRGlzaxIdCgp1c2VfbWVtb3J5GAIgASgIUgl1c2VNZW1vcnkSIAoMdXNlX29mZl9oZWFwGAMgASgIUgp1c2VPZmZIZWFwEiIKDGRlc2VyaWFsaXplZBgEIAEoCFIMZGVzZXJpYWxpemVkEiAKC3JlcGxpY2F0aW9uGAUgASgFUgtyZXBsaWNhdGlvbkIiCh5vcmcuYXBhY2hlLnNwYXJrLmNvbm5lY3QucHJvdG9QAWIGcHJvdG8z"
+ }
+ }],
+ "isInternal": false
+ }
+ },
+ "name": ["col"]
+ }
+ }]
}
},
"expressions": [{
@@ -16,7 +47,7 @@
"functionName": "to_protobuf",
"arguments": [{
"unresolvedAttribute": {
- "unparsedIdentifier": "bytes"
+ "unparsedIdentifier": "col"
}
}, {
"literal": {
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.proto.bin
index 7f955b1013fe..715c604d632d 100644
Binary files a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.proto.bin differ
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org