You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/12 11:18:01 UTC

(datafusion-comet) 01/01: test: migrate required-child hash regressions to SQL fixtures (#6864)

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6864-cf6673808dc92d0be7ed331431fe9a2e97067a7d
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git

commit 71ae2284841dbcab0efc2d6523bbfcbe8ddde8df
Author: KUAN-HAO HUANG <10...@users.noreply.github.com>
AuthorDate: Mon Oct 12 11:17:36 2026 +0000

    test: migrate required-child hash regressions to SQL fixtures (#6864)
---
 .../expressions/hash/hash_required_child.sql       | 49 +++++++++++
 .../apache/comet/CometHashExpressionSuite.scala    | 97 +---------------------
 2 files changed, 50 insertions(+), 96 deletions(-)

diff --git a/spark/src/test/resources/sql-tests/expressions/hash/hash_required_child.sql b/spark/src/test/resources/sql-tests/expressions/hash/hash_required_child.sql
new file mode 100644
index 0000000000..4f95e40cfa
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/hash/hash_required_child.sql
@@ -0,0 +1,49 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- ConfigMatrix: parquet.enable.dictionary=false,true
+
+-- CTAS preserves REQUIRED children. CREATE + INSERT can rebuild them as nullable,
+-- which would hide #5754: a child can contain a value underneath a NULL parent.
+-- Write with Spark so these tests exercise the native reader and hash kernels.
+statement
+SET spark.comet.enabled=false
+
+statement
+CREATE TABLE hash_required_child USING parquet AS SELECT /*+ COALESCE(1) */ c FROM VALUES (named_struct('a', 1)), (NULL), (named_struct('a', 3)), (NULL) AS v(c)
+
+-- The parent's null mask must also reach grandchildren through a REQUIRED struct.
+statement
+CREATE TABLE hash_required_nested_child USING parquet AS SELECT /*+ COALESCE(1) */ c FROM VALUES (named_struct('b', named_struct('x', 1))), (NULL), (named_struct('b', named_struct('x', 3))), (NULL) AS v(c)
+
+statement
+SET spark.comet.enabled=true
+
+query expect_native(hash,xxhash64)
+SELECT hash(c), xxhash64(c) FROM hash_required_child ORDER BY 1, 2
+
+query expect_native(hash,xxhash64)
+SELECT hash(c), xxhash64(c) FROM hash_required_nested_child ORDER BY 1, 2
+
+-- A valid array element wrapping the NULL struct preserves its hidden child values.
+-- A NULL array element itself is rebuilt and would not exercise the same path.
+query expect_native(hash,xxhash64)
+SELECT hash(array(named_struct('tag', 1, 'b', c))), xxhash64(array(named_struct('tag', 1, 'b', c))) FROM hash_required_child ORDER BY 1, 2
+
+-- Two elements exercise chaining the second element's hash onto the first.
+query expect_native(hash,xxhash64)
+SELECT hash(array(named_struct('tag', 1, 'b', c), named_struct('tag', 2, 'b', c))), xxhash64(array(named_struct('tag', 1, 'b', c), named_struct('tag', 2, 'b', c))) FROM hash_required_child ORDER BY 1, 2
diff --git a/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala
index bbf6c4289a..c20e5899b3 100644
--- a/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala
@@ -21,9 +21,8 @@ package org.apache.comet
 
 import scala.util.Random
 
-import org.apache.spark.sql.{CometTestBase, Row}
+import org.apache.spark.sql.CometTestBase
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
-import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
 
 import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, ParquetGenerator, SchemaGenOptions}
 
@@ -259,100 +258,6 @@ class CometHashExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelpe
     }
   }
 
-  test("hash - null struct with a required child field") {
-    // `c` is nullable but its child is REQUIRED, so Spark writes
-    // `optional group c { required int32 a; }`. On read the child leaf has nowhere to record a
-    // null of its own, so its buffer holds a value at exactly the rows where the struct is null.
-    // That is the shape where the parent's null mask has to reach the children; a struct built in
-    // the plan, or one whose child is also nullable, has null children there and hides the bug.
-    withTempPath { dir =>
-      val schema = StructType(
-        Seq(
-          StructField(
-            "c",
-            StructType(Seq(StructField("a", IntegerType, nullable = false))),
-            nullable = true)))
-      withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
-        val rows = Seq(Row(Row(1)), Row(null), Row(Row(3)), Row(null))
-        spark
-          .createDataFrame(spark.sparkContext.parallelize(rows), schema)
-          .coalesce(1)
-          .write
-          .parquet(dir.toString)
-      }
-      spark.read.parquet(dir.toString).createOrReplaceTempView("null_struct_t")
-      checkSparkAnswerAndOperator("SELECT hash(c), xxhash64(c) FROM null_struct_t ORDER BY 1, 2")
-    }
-  }
-
-  test("hash - null struct whose child is itself a struct") {
-    // The union has to recurse: the outer struct's nulls reach the inner struct, whose own
-    // children are required and so carry values under the null.
-    withTempPath { dir =>
-      val inner = StructType(Seq(StructField("x", IntegerType, nullable = false)))
-      val schema = StructType(
-        Seq(
-          StructField(
-            "c",
-            StructType(Seq(StructField("b", inner, nullable = false))),
-            nullable = true)))
-      withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
-        val rows = Seq(Row(Row(Row(1))), Row(null), Row(Row(Row(3))), Row(null))
-        spark
-          .createDataFrame(spark.sparkContext.parallelize(rows), schema)
-          .coalesce(1)
-          .write
-          .parquet(dir.toString)
-      }
-      spark.read.parquet(dir.toString).createOrReplaceTempView("null_nested_struct_t")
-      checkSparkAnswerAndOperator(
-        "SELECT hash(c), xxhash64(c) FROM null_nested_struct_t ORDER BY 1, 2")
-    }
-  }
-
-  test("hash - list element wrapping a null struct with a required child field") {
-    // The per-element path in `hash_list_array!`, which #5567 made usable as a shuffle key.
-    //
-    // Wrapping the struct rather than using it as the element directly is what makes this
-    // reachable: `array(named_struct('tag', 1, 'b', c))` produces an element that is itself
-    // valid, so it is copied rather than rebuilt, and the null `c` inside it keeps the child
-    // values Parquet wrote under it. Using `c` as the element directly does not reproduce,
-    // because a null element is rebuilt on the way in and the hidden values go with it.
-    //
-    // So this covers a null struct nested inside a valid element. The unit test in murmur3.rs
-    // covers the complementary shape, where the element itself is the null struct.
-    withTempPath { dir =>
-      val schema = StructType(
-        Seq(
-          StructField(
-            "c",
-            StructType(Seq(StructField("a", IntegerType, nullable = false))),
-            nullable = true)))
-      withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
-        val rows = Seq(Row(Row(1)), Row(null), Row(Row(3)), Row(null))
-        spark
-          .createDataFrame(spark.sparkContext.parallelize(rows), schema)
-          .coalesce(1)
-          .write
-          .parquet(dir.toString)
-      }
-      spark.read.parquet(dir.toString).createOrReplaceTempView("wrapped_null_struct_t")
-
-      checkSparkAnswerAndOperator("""
-        SELECT
-          hash(array(named_struct('tag', 1, 'b', c))),
-          xxhash64(array(named_struct('tag', 1, 'b', c)))
-        FROM wrapped_null_struct_t ORDER BY 1, 2""")
-
-      // Two elements, so the hash of the second chains onto the first.
-      checkSparkAnswerAndOperator("""
-        SELECT
-          hash(array(named_struct('tag', 1, 'b', c), named_struct('tag', 2, 'b', c))),
-          xxhash64(array(named_struct('tag', 1, 'b', c), named_struct('tag', 2, 'b', c)))
-        FROM wrapped_null_struct_t ORDER BY 1, 2""")
-    }
-  }
-
   test("hash - struct with array field") {
     withTable("t") {
       sql("CREATE TABLE t(c STRUCT<a: INT, b: ARRAY<STRING>>) USING parquet")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org