You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2024/06/27 16:12:27 UTC
(spark) branch branch-3.5 updated: [SPARK-48709][SQL][3.5] Fix varchar type resolution mismatch for DataSourceV2 CTAS
This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 3d7a20a08ec1 [SPARK-48709][SQL][3.5] Fix varchar type resolution mismatch for DataSourceV2 CTAS
3d7a20a08ec1 is described below
commit 3d7a20a08ec104e98f15fa0926547086904a0fe8
Author: Yuming Wang <yu...@ebay.com>
AuthorDate: Fri Jun 28 00:12:13 2024 +0800
[SPARK-48709][SQL][3.5] Fix varchar type resolution mismatch for DataSourceV2 CTAS
Backport of #47082.
### What changes were proposed in this pull request?
This PR fixes varchar type resolution mismatch for DataSourceV2 CTAS. For example:
```sql
set spark.sql.storeAssignmentPolicy=LEGACY;
CREATE TABLE testcat.ns.t1 (d1 string, d2 varchar(200)) USING parquet;
CREATE TABLE testcat.ns.t2 USING foo as select * from testcat.ns.t1
```
Error message:
```
org.apache.spark.sql.AnalysisException: LEGACY store assignment policy is disallowed in Spark data source V2. Please set the configuration spark.sql.storeAssignmentPolicy to other values.
```
### Why are the changes needed?
Avoid query failures.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Unit test.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #47103 from wangyum/SPARK-48709-branch-3.5.
Authored-by: Yuming Wang <yu...@ebay.com>
Signed-off-by: Kent Yao <ya...@apache.org>
---
.../apache/spark/sql/catalyst/plans/logical/v2Commands.scala | 3 ++-
.../org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala | 10 ++++++++++
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
index 739ffa487e39..805f277cf9f6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
@@ -62,10 +62,11 @@ trait V2WriteCommand extends UnaryCommand with KeepAnalyzedQuery {
table.skipSchemaResolution || (query.output.size == table.output.size &&
query.output.zip(table.output).forall {
case (inAttr, outAttr) =>
+ val inType = CharVarcharUtils.getRawType(inAttr.metadata).getOrElse(inAttr.dataType)
val outType = CharVarcharUtils.getRawType(outAttr.metadata).getOrElse(outAttr.dataType)
// names and types must match, nullability must be compatible
inAttr.name == outAttr.name &&
- DataType.equalsIgnoreCompatibleNullability(inAttr.dataType, outType) &&
+ DataType.equalsIgnoreCompatibleNullability(inType, outType) &&
(outAttr.nullable || !inAttr.nullable)
})
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index 6df71d08ef72..1db63f319728 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -1736,6 +1736,16 @@ class DataSourceV2SQLSuiteV1Filter
}
}
+ test("SPARK-48709: varchar resolution mismatch for DataSourceV2 CTAS") {
+ withSQLConf(
+ SQLConf.STORE_ASSIGNMENT_POLICY.key -> SQLConf.StoreAssignmentPolicy.LEGACY.toString) {
+ withTable("testcat.ns.t1", "testcat.ns.t2") {
+ sql("CREATE TABLE testcat.ns.t1 (d1 string, d2 varchar(200)) USING parquet")
+ sql("CREATE TABLE testcat.ns.t2 USING foo as select * from testcat.ns.t1")
+ }
+ }
+ }
+
test("ShowCurrentNamespace: basic tests") {
def testShowCurrentNamespace(expectedCatalogName: String, expectedNamespace: String): Unit = {
val schema = new StructType()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org