You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2026/04/16 09:34:08 UTC
(spark) branch master updated: revert [SPARK-43752] and add test
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5d491f62748b revert [SPARK-43752] and add test
5d491f62748b is described below
commit 5d491f62748b4b9c34bc3b5bd7390f7b5ca75053
Author: Wenchen Fan <we...@databricks.com>
AuthorDate: Thu Apr 16 17:33:53 2026 +0800
revert [SPARK-43752] and add test
### What changes were proposed in this pull request?
This PR reverts the code changes from #55127 (commit eca57ea955b) and moves the test to `InsertIntoTests` where it properly runs across V2 INSERT SQL test suites.
Specifically:
- **Removes** the `V2WriteCommand` case added to `Analyzer.ResolveReferences` and `ResolveColumnDefaultInCommandInputQuery`
- **Removes** the test from `ResolveDefaultColumnsSuite` (wrong location for a V2 end-to-end test)
- **Adds** the test to `InsertIntoSQLOnlyTests` in `InsertIntoTests.scala`, which runs in suites with `includeSQLOnlyTests = true`: `DataSourceV2SQLSuite`, `DataSourceV2SQLSessionCatalogSuite`, and `V1WriteFallbackSessionCatalogSuite`. This is correct since `DEFAULT` is SQL-only syntax.
### Why are the changes needed?
The code added by #55127 is dead code. For `INSERT INTO v2_table VALUES (1, DEFAULT)`, the resolution flow is:
1. The parser produces `InsertIntoStatement` for **both** V1 and V2 tables.
2. In the Resolution batch, `ResolveReferences` dispatches `InsertIntoStatement` to `resolveColumnDefaultInCommandInputQuery`, which resolves DEFAULT using the table schema. For V2 tables, the `DataSourceV2Relation` schema includes column default metadata (via `CatalogV2Util.encodeDefaultValue`), so DEFAULT is resolved correctly at this stage.
3. `ResolveInsertInto` only converts `InsertIntoStatement` to a `V2WriteCommand` (`AppendData`/`OverwriteByExpression`/`OverwritePartitionsDynamic`) **after** the query is fully resolved — by which point DEFAULT is already gone.
No code path (SQL or DataFrame API) creates a `V2WriteCommand` with unresolved DEFAULT attributes. The `PlanResolutionSuite` test "INSERT INTO table with default column value" already verifies this resolution at the plan level.
The test is still valuable and passes without the code changes — this PR moves it to the proper location.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Moved the existing test to `InsertIntoSQLOnlyTests` in `InsertIntoTests.scala`. The test verifies:
- `INSERT INTO` with partial DEFAULT values
- `INSERT INTO` with all DEFAULT values
- `INSERT OVERWRITE` with DEFAULT values
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code 4.6
Closes #55348 from cloud-fan/SPARK-43752-revert.
Authored-by: Wenchen Fan <we...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../spark/sql/catalyst/analysis/Analyzer.scala | 6 ------
.../ResolveColumnDefaultInCommandInputQuery.scala | 6 ------
.../spark/sql/ResolveDefaultColumnsSuite.scala | 24 ----------------------
.../spark/sql/connector/InsertIntoTests.scala | 17 +++++++++++++++
4 files changed, 17 insertions(+), 36 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 29180c34c4e5..1224b8ba18a3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1418,12 +1418,6 @@ class Analyzer(
// to resolve column "DEFAULT" in the child plans so that they must be unresolved.
case s: SetVariable => resolveColumnDefaultInCommandInputQuery(s)
- // SPARK-43752: resolve column "DEFAULT" in V2 write commands before the
- // query is fully resolved, matching the InsertIntoStatement behavior above.
- case v2: V2WriteCommand
- if v2.table.resolved && v2.query.containsPattern(UNRESOLVED_ATTRIBUTE) =>
- resolveColumnDefaultInCommandInputQuery(v2)
-
// Skip FetchCursor - let ResolveFetchCursor handle variable resolution
// This prevents ResolveReferences from trying to resolve target variables as columns
case s: SingleStatement if s.parsedPlan.isInstanceOf[FetchCursor] => s
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveColumnDefaultInCommandInputQuery.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveColumnDefaultInCommandInputQuery.scala
index 03db3fae2fc8..39fc6028ccba 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveColumnDefaultInCommandInputQuery.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveColumnDefaultInCommandInputQuery.scala
@@ -89,12 +89,6 @@ class ResolveColumnDefaultInCommandInputQuery(val catalogManager: CatalogManager
// defined for the n-th variable of the SET.
s.withNewChildren(Seq(resolveColumnDefault(s.sourceQuery, expectedQuerySchema)))
- // SPARK-43752: resolve column "DEFAULT" in V2 write commands.
- case v2: V2WriteCommand if conf.enableDefaultColumns && v2.table.resolved &&
- v2.query.containsPattern(UNRESOLVED_ATTRIBUTE) =>
- val expectedQuerySchema = v2.table.schema
- v2.withNewQuery(resolveColumnDefault(v2.query, expectedQuerySchema))
-
case _ => plan
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala
index 5ad945dd75b8..cb9d0909554b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala
@@ -308,28 +308,4 @@ class ResolveDefaultColumnsSuite extends QueryTest with SharedSparkSession {
checkAnswer(sql(s"SELECT * FROM $tableName"), Seq(Row(0, user)))
}
}
-
- test("SPARK-43752: column DEFAULT in V2 write commands") {
- withSQLConf(
- "spark.sql.catalog.testcat" ->
- classOf[connector.catalog.InMemoryTableCatalog].getName) {
- sql("CREATE TABLE testcat.t(c1 INT DEFAULT 42, c2 STRING DEFAULT 'hello')")
- // INSERT INTO (AppendData) with DEFAULT
- sql("INSERT INTO testcat.t VALUES (1, DEFAULT)")
- checkAnswer(
- sql("SELECT * FROM testcat.t"),
- Row(1, "hello"))
- // INSERT INTO with all DEFAULT
- sql("INSERT INTO testcat.t VALUES (DEFAULT, DEFAULT)")
- checkAnswer(
- sql("SELECT * FROM testcat.t ORDER BY c1"),
- Seq(Row(1, "hello"), Row(42, "hello")))
- // INSERT OVERWRITE (OverwriteByExpression) with DEFAULT
- sql("INSERT OVERWRITE testcat.t VALUES (100, DEFAULT)")
- checkAnswer(
- sql("SELECT * FROM testcat.t"),
- Row(100, "hello"))
- sql("DROP TABLE testcat.t")
- }
- }
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala
index dd0c8f0a29e9..1ad8e7e1c683 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala
@@ -505,6 +505,23 @@ trait InsertIntoSQLOnlyTests
verifyTable(t1, Seq("a" -> "b").toDF("id", "data"))
}
}
+
+ test("InsertInto: column DEFAULT values") {
+ val t1 = s"${catalogAndNamespace}tbl"
+ withTable(t1) {
+ sql(s"CREATE TABLE $t1 (c1 INT DEFAULT 42, c2 STRING DEFAULT 'hello') USING $v2Format")
+ sql(s"INSERT INTO $t1 VALUES (1, DEFAULT)")
+ checkAnswer(sql(s"SELECT * FROM $t1"), Row(1, "hello"))
+
+ sql(s"INSERT INTO $t1 VALUES (DEFAULT, DEFAULT)")
+ checkAnswer(
+ sql(s"SELECT * FROM $t1 ORDER BY c1"),
+ Seq(Row(1, "hello"), Row(42, "hello")))
+
+ sql(s"INSERT OVERWRITE $t1 VALUES (100, DEFAULT)")
+ checkAnswer(sql(s"SELECT * FROM $t1"), Row(100, "hello"))
+ }
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org