(spark) branch branch-4.0 updated: [SPARK-51284][SQL] Fix SQL Script execution for empty result

[we...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.0 by this push:
     new 66d81aab58d8 [SPARK-51284][SQL] Fix SQL Script execution for empty result
66d81aab58d8 is described below

commit 66d81aab58d84e3e73fd7e654e8ddc5a23fd97d5
Author: David Milicevic <da...@databricks.com>
AuthorDate: Fri Feb 21 20:20:15 2025 +0800

    [SPARK-51284][SQL] Fix SQL Script execution for empty result
    
    ### What changes were proposed in this pull request?
    
    This pull request proposes changes to fix the empty result returned from the SQL Script.
    Change is quite simple, it just introduces the `resultSchema` var to track the schema of the last resulting statement in the SQL Script.
    This way, the case when `result` is empty and there are no rows (and thus the schema cannot be fetched this way) is fixed, because the schema is now read from `DataFrame`.
    
    ### Why are the changes needed?
    
    Fixing a bug - if the result statement in the SQL Script exists, but returns nothing, there were few problems:
    - Tests were missing.
    - Execution was failing.
    - Schema would be missing.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Already existing tests.
    Adapted the `SqlScriptingE2eSuite` to support output schema checks.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No.
    
    Closes #50024 from davidm-db/scripting_empty_result_fix.
    
    Authored-by: David Milicevic <da...@databricks.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
    (cherry picked from commit b70b90461681764db99ea3241bb46afbf8aca202)
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../apache/spark/sql/classic/SparkSession.scala    |  7 +++++-
 .../spark/sql/scripting/SqlScriptingE2eSuite.scala | 26 +++++++++++++++++++++-
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala
index 205abf365091..14ae46c25a90 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala
@@ -453,10 +453,12 @@ class SparkSession private(
       // collect results immediately to maintain execution order.
       // This ensures we respect the contract of SqlScriptingExecution API.
       var df: Option[DataFrame] = sse.getNextResult
+      var resultSchema: Option[StructType] = None
       while (df.isDefined) {
         sse.withErrorHandling {
           // Collect results from the current DataFrame.
           result = Some(df.get.collect().toSeq)
+          resultSchema = Some(df.get.schema)
         }
         df = sse.getNextResult
       }
@@ -464,7 +466,10 @@ class SparkSession private(
       if (result.isEmpty) {
         emptyDataFrame
       } else {
-        val attributes = DataTypeUtils.toAttributes(result.get.head.schema)
+        // If `result` is defined, then `resultSchema` must be defined as well.
+        assert(resultSchema.isDefined)
+
+        val attributes = DataTypeUtils.toAttributes(resultSchema.get)
         Dataset.ofRows(
           self, LocalRelation.fromExternalRows(attributes, result.get))
       }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/scripting/SqlScriptingE2eSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/scripting/SqlScriptingE2eSuite.scala
index b12114041d0e..fe3617142d12 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/scripting/SqlScriptingE2eSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/scripting/SqlScriptingE2eSuite.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.util.QuotingUtils.toSQLConf
 import org.apache.spark.sql.exceptions.SqlScriptingException
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
 
 
 /**
@@ -35,9 +36,14 @@ import org.apache.spark.sql.test.SharedSparkSession
  */
 class SqlScriptingE2eSuite extends QueryTest with SharedSparkSession {
   // Helpers
-  private def verifySqlScriptResult(sqlText: String, expected: Seq[Row]): Unit = {
+  private def verifySqlScriptResult(
+      sqlText: String,
+      expected: Seq[Row],
+      expectedSchema: Option[StructType] = None): Unit = {
     val df = spark.sql(sqlText)
     checkAnswer(df, expected)
+
+    assert(expectedSchema.forall(_ === df.schema))
   }
 
   private def verifySqlScriptResultWithNamedParams(
@@ -139,6 +145,24 @@ class SqlScriptingE2eSuite extends QueryTest with SharedSparkSession {
     verifySqlScriptResult(sqlScript, Seq.empty)
   }
 
+  test("SPARK-51284: script with empty result") {
+    withTable("scripting_test_table") {
+      val sqlScript =
+        """
+          |BEGIN
+          |  CREATE TABLE scripting_test_table (id INT);
+          |  SELECT * FROM scripting_test_table;
+          |  DROP TABLE scripting_test_table;
+          |END
+          |""".stripMargin
+      verifySqlScriptResult(
+        sqlScript,
+        Seq.empty,
+        Some(StructType(Seq(StructField("id", IntegerType)).toArray))
+      )
+    }
+  }
+
   test("empty script") {
     val sqlScript =
       """


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org