You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vh...@apache.org on 2026/07/31 13:13:02 UTC

(hudi) branch master updated: test(spark): tail-sweep coverage for low-coverage datasource classes (#19405)

This is an automated email from the ASF dual-hosted git repository.

voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 1ba9a5849e81 test(spark): tail-sweep coverage for low-coverage datasource classes (#19405)
1ba9a5849e81 is described below

commit 1ba9a5849e812f397da85130c1dd376e4099ce60
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Fri Jul 31 06:12:54 2026 -0700

    test(spark): tail-sweep coverage for low-coverage datasource classes (#19405)
    
    * test(spark): tail-sweep coverage for low-coverage datasource classes
    
    Add unit coverage for BaseHoodieCatalystExpressionUtils order-preserving
    transformation matching, org.apache.spark.sql.avro.AvroUtils schema
    matching and validation, and FileFormatUtilsForFileGroupReader filter
    lowering. Each assertion pins the exact translated output.
    
    * Rename helper2 to helperWithOptionalGhost
    
    Mirror avroWithOptionalGhost so the optional-ghost scenario reads clearly at a glance.
    
    * test(spark): address review comments on tail-sweep coverage
    
    - rename TestCatalystExpressionOrderPreserving to TestHoodieCatalystExpressionUtils
    - pin narrowing-cast behavior with a TODO referencing #19445
    - cover the per-version ParseToDate hook via its stable 1-arg constructor
    - add GreaterThanOrEqual/LessThan/sources.And arms and an unknown-column failure pin
    - use three filters so the And fold direction is observable
    - reorder Avro fields so by-name lookup is discriminated from positional
    - cover ignoreNullable=true with a non-nullable extra Catalyst field
    - pin SQLConf case sensitivity and cover the case-sensitive resolver half
    - wrap must-not-throw calls in assertDoesNotThrow
    - mark dead vendored paths (supportsDataType, positional matching) as drift guards
    
    ---------
    
    Co-authored-by: voon <vo...@gmail.com>
---
 .../hudi/TestHoodieCatalystExpressionUtils.scala   | 107 +++++++++++
 .../TestFileFormatUtilsForFileGroupReader.scala    | 115 ++++++++++++
 .../org/apache/spark/sql/avro/TestAvroUtils.scala  | 199 +++++++++++++++++++++
 3 files changed, 421 insertions(+)

diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
new file mode 100644
index 000000000000..41638d4c4242
--- /dev/null
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi
+
+import org.apache.spark.sql.catalyst.expressions.{Add, AttributeReference, BitwiseOr, Cast, DateAdd, DateSub, Divide, Exp, Expression, Literal, Log, Lower, Multiply, ParseToDate, ShiftLeft, Sqrt, Upper}
+import org.apache.spark.sql.types.{DateType, DoubleType, IntegerType, LongType, StringType}
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+
+/**
+ * Branch coverage for the order-preserving transformation matcher in
+ * [[org.apache.spark.sql.BaseHoodieCatalystExpressionUtils]]. This is what lets data skipping map a
+ * transformed column reference back to its source attribute, so each case is pinned to the exact
+ * source [[AttributeReference]] it must recover, and non-order-preserving shapes must not match.
+ */
+class TestHoodieCatalystExpressionUtils extends SparkAdapterSupport {
+
+  private val intAttr = AttributeReference("i", IntegerType)()
+  private val strAttr = AttributeReference("s", StringType)()
+  private val dblAttr = AttributeReference("d", DoubleType)()
+  private val dateAttr = AttributeReference("dt", DateType)()
+  private val longAttr = AttributeReference("l", LongType)()
+
+  private def matched(expr: Expression): Option[AttributeReference] =
+    sparkAdapter.getCatalystExpressionUtils.tryMatchAttributeOrderingPreservingTransformation(expr)
+
+  @Test
+  def testIdentityAttributeMatches(): Unit = {
+    assertEquals(Some(intAttr), matched(intAttr))
+  }
+
+  @Test
+  def testArithmeticTransformationsPreserveOrdering(): Unit = {
+    assertEquals(Some(intAttr), matched(Add(intAttr, Literal(1))))
+    assertEquals(Some(intAttr), matched(Add(Literal(1), intAttr)))
+    assertEquals(Some(intAttr), matched(Multiply(intAttr, Literal(2))))
+    assertEquals(Some(intAttr), matched(Multiply(Literal(2), intAttr)))
+    assertEquals(Some(intAttr), matched(Divide(intAttr, Literal(2))))
+    assertEquals(Some(intAttr), matched(BitwiseOr(intAttr, Literal(1))))
+    assertEquals(Some(intAttr), matched(BitwiseOr(Literal(1), intAttr)))
+    assertEquals(Some(intAttr), matched(ShiftLeft(intAttr, Literal(1))))
+  }
+
+  @Test
+  def testUnaryMathAndStringTransformationsPreserveOrdering(): Unit = {
+    assertEquals(Some(dblAttr), matched(Exp(dblAttr)))
+    assertEquals(Some(dblAttr), matched(Log(dblAttr)))
+    assertEquals(Some(strAttr), matched(Upper(strAttr)))
+    assertEquals(Some(strAttr), matched(Lower(strAttr)))
+  }
+
+  @Test
+  def testDateTransformationsPreserveOrdering(): Unit = {
+    assertEquals(Some(dateAttr), matched(DateAdd(dateAttr, Literal(1))))
+    assertEquals(Some(dateAttr), matched(DateSub(dateAttr, Literal(1))))
+  }
+
+  @Test
+  def testDateParsingTransformationsPreserveOrdering(): Unit = {
+    // ParseToDate's case-class shape differs across Spark versions, so the matcher dispatches it
+    // through the per-version unapplyOrderPreservingDateParsing hook. The 1-arg auxiliary
+    // constructor is stable on all profiles, letting this pin that hook everywhere.
+    assertEquals(Some(strAttr), matched(new ParseToDate(strAttr)))
+  }
+
+  @Test
+  def testUpCastPreservesOrderingButNumericToStringDoesNot(): Unit = {
+    // Widening a numeric column preserves ordering, so the source attribute is recovered.
+    assertEquals(Some(intAttr), matched(Cast(intAttr, LongType)))
+    // Casting a numeric column to string can reorder values, so it must not match.
+    assertEquals(None, matched(Cast(intAttr, StringType)))
+    // TODO(#19445): a narrowing numeric cast wraps around in non-ANSI mode and does not preserve
+    // ordering, so this should be None; pinning the current (incorrect) behavior until
+    // isCastPreservingOrdering rejects it.
+    assertEquals(Some(longAttr), matched(Cast(longAttr, IntegerType)))
+  }
+
+  @Test
+  def testNestedComposition(): Unit = {
+    assertEquals(Some(intAttr), matched(Add(Multiply(intAttr, Literal(2)), Literal(3))))
+  }
+
+  @Test
+  def testNonOrderPreservingExpressionsDoNotMatch(): Unit = {
+    // A bare literal carries no attribute.
+    assertEquals(None, matched(Literal(5)))
+    // No attribute on either operand.
+    assertEquals(None, matched(Add(Literal(1), Literal(2))))
+    // Sqrt is not one of the whitelisted order-preserving transformations.
+    assertEquals(None, matched(Sqrt(dblAttr)))
+  }
+}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/TestFileFormatUtilsForFileGroupReader.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/TestFileFormatUtilsForFileGroupReader.scala
new file mode 100644
index 000000000000..58457f534d90
--- /dev/null
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/TestFileFormatUtilsForFileGroupReader.scala
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeReference, Contains, EndsWith, EqualNullSafe, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, In, IsNotNull, IsNull, LessThan, LessThanOrEqual, Literal, Not, Or, StartsWith}
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LocalRelation}
+import org.apache.spark.sql.types.{BooleanType, IntegerType, StringType, StructType}
+import org.junit.jupiter.api.Assertions.{assertEquals, assertSame, assertThrows}
+import org.junit.jupiter.api.Test
+
+/**
+ * Coverage for [[FileFormatUtilsForFileGroupReader.applyFiltersToPlan]], which lowers pushed-down
+ * data-source [[org.apache.spark.sql.sources.Filter]]s into a Catalyst [[Filter]] on the plan. Each
+ * case pins the exact translated Catalyst expression so a wrong mapping would fail, and the
+ * no-filter case must return the input plan untouched.
+ */
+class TestFileFormatUtilsForFileGroupReader {
+
+  private val a: Attribute = AttributeReference("a", IntegerType)()
+  private val b: Attribute = AttributeReference("b", StringType)()
+  private val tableSchema: StructType = new StructType().add("a", IntegerType).add("b", StringType)
+  private val resolved: Seq[Attribute] = Seq(a, b)
+  private val plan: LocalRelation = LocalRelation(a, b)
+
+  private def condOf(filters: Seq[sources.Filter]): Expression =
+    FileFormatUtilsForFileGroupReader.applyFiltersToPlan(plan, tableSchema, resolved, filters) match {
+      case Filter(cond, child) =>
+        assertSame(plan, child)
+        cond
+      case other => throw new AssertionError(s"expected a Filter, got $other")
+    }
+
+  @Test
+  def testComparisonAndNullFilters(): Unit = {
+    assertEquals(EqualTo(a, Literal(1)), condOf(Seq(sources.EqualTo("a", 1))))
+    assertEquals(EqualNullSafe(a, Literal(1)), condOf(Seq(sources.EqualNullSafe("a", 1))))
+    assertEquals(GreaterThan(a, Literal(5)), condOf(Seq(sources.GreaterThan("a", 5))))
+    // GreaterThanOrEqual is the comparison incremental reads actually push down
+    // (MergeOnReadIncrementalRelationV1.incrementalSpanRecordFilters).
+    assertEquals(GreaterThanOrEqual(a, Literal(5)), condOf(Seq(sources.GreaterThanOrEqual("a", 5))))
+    assertEquals(LessThan(a, Literal(5)), condOf(Seq(sources.LessThan("a", 5))))
+    assertEquals(LessThanOrEqual(a, Literal(5)), condOf(Seq(sources.LessThanOrEqual("a", 5))))
+    assertEquals(IsNull(b), condOf(Seq(sources.IsNull("b"))))
+    assertEquals(IsNotNull(b), condOf(Seq(sources.IsNotNull("b"))))
+  }
+
+  @Test
+  def testStringAndInFilters(): Unit = {
+    assertEquals(Contains(b, Literal("x")), condOf(Seq(sources.StringContains("b", "x"))))
+    assertEquals(StartsWith(b, Literal("x")), condOf(Seq(sources.StringStartsWith("b", "x"))))
+    assertEquals(EndsWith(b, Literal("x")), condOf(Seq(sources.StringEndsWith("b", "x"))))
+    assertEquals(
+      In(a, Seq(Literal(1), Literal(2), Literal(3))),
+      condOf(Seq(sources.In("a", Array[Any](1, 2, 3)))))
+  }
+
+  @Test
+  def testConstantFilters(): Unit = {
+    assertEquals(Literal(true, BooleanType), condOf(Seq(sources.AlwaysTrue())))
+    assertEquals(Literal(false, BooleanType), condOf(Seq(sources.AlwaysFalse())))
+  }
+
+  @Test
+  def testCompositeFilters(): Unit = {
+    // A nested and/or/not tree is lowered structurally.
+    assertEquals(
+      Or(EqualTo(a, Literal(1)), Not(IsNull(b))),
+      condOf(Seq(sources.Or(sources.EqualTo("a", 1), sources.Not(sources.IsNull("b"))))))
+    // An explicit sources.And lowers through its own translate arm.
+    assertEquals(
+      And(EqualTo(a, Literal(1)), IsNull(b)),
+      condOf(Seq(sources.And(sources.EqualTo("a", 1), sources.IsNull("b")))))
+  }
+
+  @Test
+  def testMultipleFiltersAreAndedInOrder(): Unit = {
+    // Three filters (the shape incremental reads emit) make the left-nesting observable;
+    // with only two, reduceLeft and reduceRight produce the same tree.
+    assertEquals(
+      And(And(IsNotNull(b), GreaterThan(a, Literal(5))), LessThanOrEqual(a, Literal(9))),
+      condOf(Seq(sources.IsNotNull("b"), sources.GreaterThan("a", 5), sources.LessThanOrEqual("a", 9))))
+  }
+
+  @Test
+  def testFilterOnMissingColumnThrows(): Unit = {
+    // A filter naming a column absent from the table schema makes toRef miss, and the bare
+    // .get surfaces as NoSuchElementException. Reachable via readStream.schema(...) omitting
+    // the meta columns while incremental relations inject _hoodie_commit_time filters.
+    assertThrows(classOf[NoSuchElementException],
+      () => FileFormatUtilsForFileGroupReader.applyFiltersToPlan(
+        plan, tableSchema, resolved, Seq(sources.EqualTo("missing", 1))))
+  }
+
+  @Test
+  def testNoFiltersReturnsPlanUnchanged(): Unit = {
+    val result = FileFormatUtilsForFileGroupReader.applyFiltersToPlan(
+      plan, tableSchema, resolved, Seq.empty)
+    assertSame(plan, result)
+  }
+}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestAvroUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestAvroUtils.scala
new file mode 100644
index 000000000000..1447f2498d7e
--- /dev/null
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestAvroUtils.scala
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.avro
+
+import org.apache.avro.Schema
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+import org.junit.jupiter.api.Assertions.{assertDoesNotThrow, assertEquals, assertFalse, assertThrows, assertTrue}
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.function.Executable
+
+/**
+ * Direct coverage for [[AvroUtils]] and its [[AvroUtils.AvroSchemaHelper]], which are otherwise only
+ * exercised indirectly through the Avro serializers. Focuses on the type-support predicate and the
+ * schema-matching/validation error paths (extra Catalyst fields, extra required Avro fields,
+ * positional vs by-name matching, ambiguous by-name lookup), pinning the raised exception messages.
+ */
+class TestAvroUtils {
+
+  private def parse(json: String): Schema = new Schema.Parser().parse(json)
+
+  @Test
+  def testSupportsDataType(): Unit = {
+    // supportsDataType has no live Hudi caller; this guards the vendored copy against edit drift.
+    assertTrue(AvroUtils.supportsDataType(IntegerType))
+    assertTrue(AvroUtils.supportsDataType(StringType))
+    assertTrue(AvroUtils.supportsDataType(NullType))
+    assertTrue(AvroUtils.supportsDataType(ArrayType(LongType)))
+    assertTrue(AvroUtils.supportsDataType(MapType(StringType, IntegerType)))
+    assertTrue(AvroUtils.supportsDataType(
+      new StructType().add("a", IntegerType).add("b", ArrayType(StringType))))
+    // CalendarInterval is not representable in Avro, so every wrapper around it is unsupported too.
+    assertFalse(AvroUtils.supportsDataType(CalendarIntervalType))
+    assertFalse(AvroUtils.supportsDataType(ArrayType(CalendarIntervalType)))
+    assertFalse(AvroUtils.supportsDataType(MapType(StringType, CalendarIntervalType)))
+    assertFalse(AvroUtils.supportsDataType(new StructType().add("a", CalendarIntervalType)))
+  }
+
+  @Test
+  def testToFieldStr(): Unit = {
+    assertEquals("top-level record", AvroUtils.toFieldStr(Seq.empty))
+    assertEquals("field 'foo'", AvroUtils.toFieldStr(Seq("foo")))
+    assertEquals("field 'foo.bar'", AvroUtils.toFieldStr(Seq("foo", "bar")))
+  }
+
+  @Test
+  def testIsNullable(): Unit = {
+    val record = parse(
+      """{"type":"record","name":"r","fields":[
+        |  {"name":"req","type":"int"},
+        |  {"name":"opt","type":["null","int"]},
+        |  {"name":"uni","type":["int","long"]}
+        |]}""".stripMargin)
+    assertFalse(AvroUtils.isNullable(record.getField("req")))
+    assertTrue(AvroUtils.isNullable(record.getField("opt")))
+    // A union without a NULL branch is not nullable.
+    assertFalse(AvroUtils.isNullable(record.getField("uni")))
+  }
+
+  @Test
+  def testSchemaHelperRejectsNonRecordSchema(): Unit = {
+    val ex = assertThrows(classOf[IncompatibleSchemaException],
+      () => new AvroUtils.AvroSchemaHelper(
+        Schema.create(Schema.Type.INT), new StructType(), Seq.empty, Seq.empty, false))
+    assertTrue(ex.getMessage.contains("as a RECORD"))
+  }
+
+  @Test
+  def testMatchedFieldsAndGetAvroFieldByName(): Unit = {
+    // Avro order differs from Catalyst so by-name lookup is discriminated from positional.
+    val avro = parse(
+      """{"type":"record","name":"r","fields":[
+        |  {"name":"name","type":["null","string"]},
+        |  {"name":"id","type":"int"}
+        |]}""".stripMargin)
+    val catalyst = new StructType().add("id", IntegerType).add("name", StringType)
+    val helper = new AvroUtils.AvroSchemaHelper(avro, catalyst, Seq.empty, Seq.empty, false)
+
+    assertEquals(2, helper.matchedFields.size)
+    assertEquals("id", helper.getAvroField("id", 0).get.name())
+    assertTrue(helper.getAvroField("missing", 5).isEmpty)
+  }
+
+  @Test
+  def testGetAvroFieldPositional(): Unit = {
+    val avro = parse(
+      """{"type":"record","name":"r","fields":[{"name":"only","type":"int"}]}""")
+    val helper = new AvroUtils.AvroSchemaHelper(avro, new StructType(), Seq.empty, Seq.empty, true)
+    // Hudi never enables positionalFieldMatch; vendored-path drift guard.
+    // Positional matching ignores the name and selects by index.
+    assertEquals("only", helper.getAvroField("anything", 0).get.name())
+    assertTrue(helper.getAvroField("anything", 1).isEmpty)
+  }
+
+  @Test
+  def testValidateNoExtraCatalystFieldsByName(): Unit = {
+    val avro = parse(
+      """{"type":"record","name":"r","fields":[{"name":"id","type":"int"}]}""")
+    // A nullable Catalyst field with no Avro counterpart.
+    val catalyst = new StructType().add("id", IntegerType).add("extra", StringType, nullable = true)
+    val helper = new AvroUtils.AvroSchemaHelper(avro, catalyst, Seq.empty, Seq.empty, false)
+
+    val ex = assertThrows(classOf[IncompatibleSchemaException],
+      () => helper.validateNoExtraCatalystFields(ignoreNullable = false))
+    assertTrue(ex.getMessage.contains("Cannot find field 'extra' in Avro schema"))
+
+    // When nullable Catalyst fields are ignored, the same extra field is tolerated.
+    assertDoesNotThrow(new Executable {
+      def execute(): Unit = helper.validateNoExtraCatalystFields(ignoreNullable = true)
+    })
+
+    // A non-nullable extra Catalyst field must still throw even when nullables are ignored;
+    // this is the combination the deserializer relies on (ignoreNullable = true).
+    val strictCatalyst = new StructType().add("id", IntegerType).add("extra", StringType, nullable = false)
+    val strictHelper = new AvroUtils.AvroSchemaHelper(avro, strictCatalyst, Seq.empty, Seq.empty, false)
+    val strictEx = assertThrows(classOf[IncompatibleSchemaException],
+      () => strictHelper.validateNoExtraCatalystFields(ignoreNullable = true))
+    assertTrue(strictEx.getMessage.contains("Cannot find field 'extra' in Avro schema"))
+  }
+
+  @Test
+  def testValidateNoExtraCatalystFieldsPositional(): Unit = {
+    // Hudi never enables positionalFieldMatch; vendored-path drift guard.
+    val avro = parse(
+      """{"type":"record","name":"r","fields":[{"name":"id","type":"int"}]}""")
+    val catalyst = new StructType().add("id", IntegerType).add("second", IntegerType)
+    val helper = new AvroUtils.AvroSchemaHelper(avro, catalyst, Seq.empty, Seq.empty, true)
+
+    val ex = assertThrows(classOf[IncompatibleSchemaException],
+      () => helper.validateNoExtraCatalystFields(ignoreNullable = false))
+    assertTrue(ex.getMessage.contains("Cannot find field at position 1"))
+  }
+
+  @Test
+  def testValidateNoExtraRequiredAvroFields(): Unit = {
+    val avroWithRequiredGhost = parse(
+      """{"type":"record","name":"r","fields":[
+        |  {"name":"id","type":"int"},
+        |  {"name":"ghost","type":"int"}
+        |]}""".stripMargin)
+    val catalyst = new StructType().add("id", IntegerType)
+    val helper = new AvroUtils.AvroSchemaHelper(
+      avroWithRequiredGhost, catalyst, Seq.empty, Seq.empty, false)
+    val ex = assertThrows(classOf[IncompatibleSchemaException],
+      () => helper.validateNoExtraRequiredAvroFields())
+    assertTrue(ex.getMessage.contains("Found field 'ghost'"))
+
+    // A nullable extra Avro field is not required, so it is tolerated.
+    val avroWithOptionalGhost = parse(
+      """{"type":"record","name":"r","fields":[
+        |  {"name":"id","type":"int"},
+        |  {"name":"ghost","type":["null","int"]}
+        |]}""".stripMargin)
+    val helperWithOptionalGhost = new AvroUtils.AvroSchemaHelper(
+      avroWithOptionalGhost, catalyst, Seq.empty, Seq.empty, false)
+    assertDoesNotThrow(new Executable {
+      def execute(): Unit = helperWithOptionalGhost.validateNoExtraRequiredAvroFields()
+    })
+  }
+
+  @Test
+  def testGetFieldByNameAmbiguousMatch(): Unit = {
+    // Two fields differing only by case collide under a case-insensitive resolver.
+    val avro = parse(
+      """{"type":"record","name":"r","fields":[
+        |  {"name":"ID","type":"int"},
+        |  {"name":"id","type":"int"}
+        |]}""".stripMargin)
+    val helper = new AvroUtils.AvroSchemaHelper(avro, new StructType(), Seq.empty, Seq.empty, false)
+    // The resolver comes from ambient SQLConf, so pin it instead of relying on the JVM default.
+    val conf = SQLConf.get
+    conf.setConfString("spark.sql.caseSensitive", "false")
+    try {
+      val ex = assertThrows(classOf[IncompatibleSchemaException],
+        () => helper.getFieldByName("id"))
+      assertTrue(ex.getMessage.contains("gave 2 matches"))
+      // Under a case-sensitive resolver the same lookup is unambiguous.
+      conf.setConfString("spark.sql.caseSensitive", "true")
+      assertEquals("id", helper.getFieldByName("id").get.name())
+    } finally {
+      conf.unsetConf("spark.sql.caseSensitive")
+    }
+  }
+}