You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vh...@apache.org on 2026/07/31 13:13:02 UTC
(hudi) branch master updated: test(spark): tail-sweep coverage for low-coverage datasource classes (#19405)
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 1ba9a5849e81 test(spark): tail-sweep coverage for low-coverage datasource classes (#19405)
1ba9a5849e81 is described below
commit 1ba9a5849e812f397da85130c1dd376e4099ce60
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Fri Jul 31 06:12:54 2026 -0700
test(spark): tail-sweep coverage for low-coverage datasource classes (#19405)
* test(spark): tail-sweep coverage for low-coverage datasource classes
Add unit coverage for BaseHoodieCatalystExpressionUtils order-preserving
transformation matching, org.apache.spark.sql.avro.AvroUtils schema
matching and validation, and FileFormatUtilsForFileGroupReader filter
lowering. Each assertion pins the exact translated output.
* Rename helper2 to helperWithOptionalGhost
Mirror avroWithOptionalGhost so the optional-ghost scenario reads clearly at a glance.
* test(spark): address review comments on tail-sweep coverage
- rename TestCatalystExpressionOrderPreserving to TestHoodieCatalystExpressionUtils
- pin narrowing-cast behavior with a TODO referencing #19445
- cover the per-version ParseToDate hook via its stable 1-arg constructor
- add GreaterThanOrEqual/LessThan/sources.And arms and an unknown-column failure pin
- use three filters so the And fold direction is observable
- reorder Avro fields so by-name lookup is discriminated from positional
- cover ignoreNullable=true with a non-nullable extra Catalyst field
- pin SQLConf case sensitivity and cover the case-sensitive resolver half
- wrap must-not-throw calls in assertDoesNotThrow
- mark dead vendored paths (supportsDataType, positional matching) as drift guards
---------
Co-authored-by: voon <vo...@gmail.com>
---
.../hudi/TestHoodieCatalystExpressionUtils.scala | 107 +++++++++++
.../TestFileFormatUtilsForFileGroupReader.scala | 115 ++++++++++++
.../org/apache/spark/sql/avro/TestAvroUtils.scala | 199 +++++++++++++++++++++
3 files changed, 421 insertions(+)
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
new file mode 100644
index 000000000000..41638d4c4242
--- /dev/null
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi
+
+import org.apache.spark.sql.catalyst.expressions.{Add, AttributeReference, BitwiseOr, Cast, DateAdd, DateSub, Divide, Exp, Expression, Literal, Log, Lower, Multiply, ParseToDate, ShiftLeft, Sqrt, Upper}
+import org.apache.spark.sql.types.{DateType, DoubleType, IntegerType, LongType, StringType}
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+
+/**
+ * Branch coverage for the order-preserving transformation matcher in
+ * [[org.apache.spark.sql.BaseHoodieCatalystExpressionUtils]]. This is what lets data skipping map a
+ * transformed column reference back to its source attribute, so each case is pinned to the exact
+ * source [[AttributeReference]] it must recover, and non-order-preserving shapes must not match.
+ */
+class TestHoodieCatalystExpressionUtils extends SparkAdapterSupport {
+
+ private val intAttr = AttributeReference("i", IntegerType)()
+ private val strAttr = AttributeReference("s", StringType)()
+ private val dblAttr = AttributeReference("d", DoubleType)()
+ private val dateAttr = AttributeReference("dt", DateType)()
+ private val longAttr = AttributeReference("l", LongType)()
+
+ private def matched(expr: Expression): Option[AttributeReference] =
+ sparkAdapter.getCatalystExpressionUtils.tryMatchAttributeOrderingPreservingTransformation(expr)
+
+ @Test
+ def testIdentityAttributeMatches(): Unit = {
+ assertEquals(Some(intAttr), matched(intAttr))
+ }
+
+ @Test
+ def testArithmeticTransformationsPreserveOrdering(): Unit = {
+ assertEquals(Some(intAttr), matched(Add(intAttr, Literal(1))))
+ assertEquals(Some(intAttr), matched(Add(Literal(1), intAttr)))
+ assertEquals(Some(intAttr), matched(Multiply(intAttr, Literal(2))))
+ assertEquals(Some(intAttr), matched(Multiply(Literal(2), intAttr)))
+ assertEquals(Some(intAttr), matched(Divide(intAttr, Literal(2))))
+ assertEquals(Some(intAttr), matched(BitwiseOr(intAttr, Literal(1))))
+ assertEquals(Some(intAttr), matched(BitwiseOr(Literal(1), intAttr)))
+ assertEquals(Some(intAttr), matched(ShiftLeft(intAttr, Literal(1))))
+ }
+
+ @Test
+ def testUnaryMathAndStringTransformationsPreserveOrdering(): Unit = {
+ assertEquals(Some(dblAttr), matched(Exp(dblAttr)))
+ assertEquals(Some(dblAttr), matched(Log(dblAttr)))
+ assertEquals(Some(strAttr), matched(Upper(strAttr)))
+ assertEquals(Some(strAttr), matched(Lower(strAttr)))
+ }
+
+ @Test
+ def testDateTransformationsPreserveOrdering(): Unit = {
+ assertEquals(Some(dateAttr), matched(DateAdd(dateAttr, Literal(1))))
+ assertEquals(Some(dateAttr), matched(DateSub(dateAttr, Literal(1))))
+ }
+
+ @Test
+ def testDateParsingTransformationsPreserveOrdering(): Unit = {
+ // ParseToDate's case-class shape differs across Spark versions, so the matcher dispatches it
+ // through the per-version unapplyOrderPreservingDateParsing hook. The 1-arg auxiliary
+ // constructor is stable on all profiles, letting this pin that hook everywhere.
+ assertEquals(Some(strAttr), matched(new ParseToDate(strAttr)))
+ }
+
+ @Test
+ def testUpCastPreservesOrderingButNumericToStringDoesNot(): Unit = {
+ // Widening a numeric column preserves ordering, so the source attribute is recovered.
+ assertEquals(Some(intAttr), matched(Cast(intAttr, LongType)))
+ // Casting a numeric column to string can reorder values, so it must not match.
+ assertEquals(None, matched(Cast(intAttr, StringType)))
+ // TODO(#19445): a narrowing numeric cast wraps around in non-ANSI mode and does not preserve
+ // ordering, so this should be None; pinning the current (incorrect) behavior until
+ // isCastPreservingOrdering rejects it.
+ assertEquals(Some(longAttr), matched(Cast(longAttr, IntegerType)))
+ }
+
+ @Test
+ def testNestedComposition(): Unit = {
+ assertEquals(Some(intAttr), matched(Add(Multiply(intAttr, Literal(2)), Literal(3))))
+ }
+
+ @Test
+ def testNonOrderPreservingExpressionsDoNotMatch(): Unit = {
+ // A bare literal carries no attribute.
+ assertEquals(None, matched(Literal(5)))
+ // No attribute on either operand.
+ assertEquals(None, matched(Add(Literal(1), Literal(2))))
+ // Sqrt is not one of the whitelisted order-preserving transformations.
+ assertEquals(None, matched(Sqrt(dblAttr)))
+ }
+}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/TestFileFormatUtilsForFileGroupReader.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/TestFileFormatUtilsForFileGroupReader.scala
new file mode 100644
index 000000000000..58457f534d90
--- /dev/null
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/TestFileFormatUtilsForFileGroupReader.scala
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeReference, Contains, EndsWith, EqualNullSafe, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, In, IsNotNull, IsNull, LessThan, LessThanOrEqual, Literal, Not, Or, StartsWith}
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LocalRelation}
+import org.apache.spark.sql.types.{BooleanType, IntegerType, StringType, StructType}
+import org.junit.jupiter.api.Assertions.{assertEquals, assertSame, assertThrows}
+import org.junit.jupiter.api.Test
+
+/**
+ * Coverage for [[FileFormatUtilsForFileGroupReader.applyFiltersToPlan]], which lowers pushed-down
+ * data-source [[org.apache.spark.sql.sources.Filter]]s into a Catalyst [[Filter]] on the plan. Each
+ * case pins the exact translated Catalyst expression so a wrong mapping would fail, and the
+ * no-filter case must return the input plan untouched.
+ */
+class TestFileFormatUtilsForFileGroupReader {
+
+ private val a: Attribute = AttributeReference("a", IntegerType)()
+ private val b: Attribute = AttributeReference("b", StringType)()
+ private val tableSchema: StructType = new StructType().add("a", IntegerType).add("b", StringType)
+ private val resolved: Seq[Attribute] = Seq(a, b)
+ private val plan: LocalRelation = LocalRelation(a, b)
+
+ private def condOf(filters: Seq[sources.Filter]): Expression =
+ FileFormatUtilsForFileGroupReader.applyFiltersToPlan(plan, tableSchema, resolved, filters) match {
+ case Filter(cond, child) =>
+ assertSame(plan, child)
+ cond
+ case other => throw new AssertionError(s"expected a Filter, got $other")
+ }
+
+ @Test
+ def testComparisonAndNullFilters(): Unit = {
+ assertEquals(EqualTo(a, Literal(1)), condOf(Seq(sources.EqualTo("a", 1))))
+ assertEquals(EqualNullSafe(a, Literal(1)), condOf(Seq(sources.EqualNullSafe("a", 1))))
+ assertEquals(GreaterThan(a, Literal(5)), condOf(Seq(sources.GreaterThan("a", 5))))
+ // GreaterThanOrEqual is the comparison incremental reads actually push down
+ // (MergeOnReadIncrementalRelationV1.incrementalSpanRecordFilters).
+ assertEquals(GreaterThanOrEqual(a, Literal(5)), condOf(Seq(sources.GreaterThanOrEqual("a", 5))))
+ assertEquals(LessThan(a, Literal(5)), condOf(Seq(sources.LessThan("a", 5))))
+ assertEquals(LessThanOrEqual(a, Literal(5)), condOf(Seq(sources.LessThanOrEqual("a", 5))))
+ assertEquals(IsNull(b), condOf(Seq(sources.IsNull("b"))))
+ assertEquals(IsNotNull(b), condOf(Seq(sources.IsNotNull("b"))))
+ }
+
+ @Test
+ def testStringAndInFilters(): Unit = {
+ assertEquals(Contains(b, Literal("x")), condOf(Seq(sources.StringContains("b", "x"))))
+ assertEquals(StartsWith(b, Literal("x")), condOf(Seq(sources.StringStartsWith("b", "x"))))
+ assertEquals(EndsWith(b, Literal("x")), condOf(Seq(sources.StringEndsWith("b", "x"))))
+ assertEquals(
+ In(a, Seq(Literal(1), Literal(2), Literal(3))),
+ condOf(Seq(sources.In("a", Array[Any](1, 2, 3)))))
+ }
+
+ @Test
+ def testConstantFilters(): Unit = {
+ assertEquals(Literal(true, BooleanType), condOf(Seq(sources.AlwaysTrue())))
+ assertEquals(Literal(false, BooleanType), condOf(Seq(sources.AlwaysFalse())))
+ }
+
+ @Test
+ def testCompositeFilters(): Unit = {
+ // A nested and/or/not tree is lowered structurally.
+ assertEquals(
+ Or(EqualTo(a, Literal(1)), Not(IsNull(b))),
+ condOf(Seq(sources.Or(sources.EqualTo("a", 1), sources.Not(sources.IsNull("b"))))))
+ // An explicit sources.And lowers through its own translate arm.
+ assertEquals(
+ And(EqualTo(a, Literal(1)), IsNull(b)),
+ condOf(Seq(sources.And(sources.EqualTo("a", 1), sources.IsNull("b")))))
+ }
+
+ @Test
+ def testMultipleFiltersAreAndedInOrder(): Unit = {
+ // Three filters (the shape incremental reads emit) make the left-nesting observable;
+ // with only two, reduceLeft and reduceRight produce the same tree.
+ assertEquals(
+ And(And(IsNotNull(b), GreaterThan(a, Literal(5))), LessThanOrEqual(a, Literal(9))),
+ condOf(Seq(sources.IsNotNull("b"), sources.GreaterThan("a", 5), sources.LessThanOrEqual("a", 9))))
+ }
+
+ @Test
+ def testFilterOnMissingColumnThrows(): Unit = {
+ // A filter naming a column absent from the table schema makes toRef miss, and the bare
+ // .get surfaces as NoSuchElementException. Reachable via readStream.schema(...) omitting
+ // the meta columns while incremental relations inject _hoodie_commit_time filters.
+ assertThrows(classOf[NoSuchElementException],
+ () => FileFormatUtilsForFileGroupReader.applyFiltersToPlan(
+ plan, tableSchema, resolved, Seq(sources.EqualTo("missing", 1))))
+ }
+
+ @Test
+ def testNoFiltersReturnsPlanUnchanged(): Unit = {
+ val result = FileFormatUtilsForFileGroupReader.applyFiltersToPlan(
+ plan, tableSchema, resolved, Seq.empty)
+ assertSame(plan, result)
+ }
+}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestAvroUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestAvroUtils.scala
new file mode 100644
index 000000000000..1447f2498d7e
--- /dev/null
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestAvroUtils.scala
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.avro
+
+import org.apache.avro.Schema
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types._
+import org.junit.jupiter.api.Assertions.{assertDoesNotThrow, assertEquals, assertFalse, assertThrows, assertTrue}
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.function.Executable
+
+/**
+ * Direct coverage for [[AvroUtils]] and its [[AvroUtils.AvroSchemaHelper]], which are otherwise only
+ * exercised indirectly through the Avro serializers. Focuses on the type-support predicate and the
+ * schema-matching/validation error paths (extra Catalyst fields, extra required Avro fields,
+ * positional vs by-name matching, ambiguous by-name lookup), pinning the raised exception messages.
+ */
+class TestAvroUtils {
+
+ private def parse(json: String): Schema = new Schema.Parser().parse(json)
+
+ @Test
+ def testSupportsDataType(): Unit = {
+ // supportsDataType has no live Hudi caller; this guards the vendored copy against edit drift.
+ assertTrue(AvroUtils.supportsDataType(IntegerType))
+ assertTrue(AvroUtils.supportsDataType(StringType))
+ assertTrue(AvroUtils.supportsDataType(NullType))
+ assertTrue(AvroUtils.supportsDataType(ArrayType(LongType)))
+ assertTrue(AvroUtils.supportsDataType(MapType(StringType, IntegerType)))
+ assertTrue(AvroUtils.supportsDataType(
+ new StructType().add("a", IntegerType).add("b", ArrayType(StringType))))
+ // CalendarInterval is not representable in Avro, so every wrapper around it is unsupported too.
+ assertFalse(AvroUtils.supportsDataType(CalendarIntervalType))
+ assertFalse(AvroUtils.supportsDataType(ArrayType(CalendarIntervalType)))
+ assertFalse(AvroUtils.supportsDataType(MapType(StringType, CalendarIntervalType)))
+ assertFalse(AvroUtils.supportsDataType(new StructType().add("a", CalendarIntervalType)))
+ }
+
+ @Test
+ def testToFieldStr(): Unit = {
+ assertEquals("top-level record", AvroUtils.toFieldStr(Seq.empty))
+ assertEquals("field 'foo'", AvroUtils.toFieldStr(Seq("foo")))
+ assertEquals("field 'foo.bar'", AvroUtils.toFieldStr(Seq("foo", "bar")))
+ }
+
+ @Test
+ def testIsNullable(): Unit = {
+ val record = parse(
+ """{"type":"record","name":"r","fields":[
+ | {"name":"req","type":"int"},
+ | {"name":"opt","type":["null","int"]},
+ | {"name":"uni","type":["int","long"]}
+ |]}""".stripMargin)
+ assertFalse(AvroUtils.isNullable(record.getField("req")))
+ assertTrue(AvroUtils.isNullable(record.getField("opt")))
+ // A union without a NULL branch is not nullable.
+ assertFalse(AvroUtils.isNullable(record.getField("uni")))
+ }
+
+ @Test
+ def testSchemaHelperRejectsNonRecordSchema(): Unit = {
+ val ex = assertThrows(classOf[IncompatibleSchemaException],
+ () => new AvroUtils.AvroSchemaHelper(
+ Schema.create(Schema.Type.INT), new StructType(), Seq.empty, Seq.empty, false))
+ assertTrue(ex.getMessage.contains("as a RECORD"))
+ }
+
+ @Test
+ def testMatchedFieldsAndGetAvroFieldByName(): Unit = {
+ // Avro order differs from Catalyst so by-name lookup is discriminated from positional.
+ val avro = parse(
+ """{"type":"record","name":"r","fields":[
+ | {"name":"name","type":["null","string"]},
+ | {"name":"id","type":"int"}
+ |]}""".stripMargin)
+ val catalyst = new StructType().add("id", IntegerType).add("name", StringType)
+ val helper = new AvroUtils.AvroSchemaHelper(avro, catalyst, Seq.empty, Seq.empty, false)
+
+ assertEquals(2, helper.matchedFields.size)
+ assertEquals("id", helper.getAvroField("id", 0).get.name())
+ assertTrue(helper.getAvroField("missing", 5).isEmpty)
+ }
+
+ @Test
+ def testGetAvroFieldPositional(): Unit = {
+ val avro = parse(
+ """{"type":"record","name":"r","fields":[{"name":"only","type":"int"}]}""")
+ val helper = new AvroUtils.AvroSchemaHelper(avro, new StructType(), Seq.empty, Seq.empty, true)
+ // Hudi never enables positionalFieldMatch; vendored-path drift guard.
+ // Positional matching ignores the name and selects by index.
+ assertEquals("only", helper.getAvroField("anything", 0).get.name())
+ assertTrue(helper.getAvroField("anything", 1).isEmpty)
+ }
+
+ @Test
+ def testValidateNoExtraCatalystFieldsByName(): Unit = {
+ val avro = parse(
+ """{"type":"record","name":"r","fields":[{"name":"id","type":"int"}]}""")
+ // A nullable Catalyst field with no Avro counterpart.
+ val catalyst = new StructType().add("id", IntegerType).add("extra", StringType, nullable = true)
+ val helper = new AvroUtils.AvroSchemaHelper(avro, catalyst, Seq.empty, Seq.empty, false)
+
+ val ex = assertThrows(classOf[IncompatibleSchemaException],
+ () => helper.validateNoExtraCatalystFields(ignoreNullable = false))
+ assertTrue(ex.getMessage.contains("Cannot find field 'extra' in Avro schema"))
+
+ // When nullable Catalyst fields are ignored, the same extra field is tolerated.
+ assertDoesNotThrow(new Executable {
+ def execute(): Unit = helper.validateNoExtraCatalystFields(ignoreNullable = true)
+ })
+
+ // A non-nullable extra Catalyst field must still throw even when nullables are ignored;
+ // this is the combination the deserializer relies on (ignoreNullable = true).
+ val strictCatalyst = new StructType().add("id", IntegerType).add("extra", StringType, nullable = false)
+ val strictHelper = new AvroUtils.AvroSchemaHelper(avro, strictCatalyst, Seq.empty, Seq.empty, false)
+ val strictEx = assertThrows(classOf[IncompatibleSchemaException],
+ () => strictHelper.validateNoExtraCatalystFields(ignoreNullable = true))
+ assertTrue(strictEx.getMessage.contains("Cannot find field 'extra' in Avro schema"))
+ }
+
+ @Test
+ def testValidateNoExtraCatalystFieldsPositional(): Unit = {
+ // Hudi never enables positionalFieldMatch; vendored-path drift guard.
+ val avro = parse(
+ """{"type":"record","name":"r","fields":[{"name":"id","type":"int"}]}""")
+ val catalyst = new StructType().add("id", IntegerType).add("second", IntegerType)
+ val helper = new AvroUtils.AvroSchemaHelper(avro, catalyst, Seq.empty, Seq.empty, true)
+
+ val ex = assertThrows(classOf[IncompatibleSchemaException],
+ () => helper.validateNoExtraCatalystFields(ignoreNullable = false))
+ assertTrue(ex.getMessage.contains("Cannot find field at position 1"))
+ }
+
+ @Test
+ def testValidateNoExtraRequiredAvroFields(): Unit = {
+ val avroWithRequiredGhost = parse(
+ """{"type":"record","name":"r","fields":[
+ | {"name":"id","type":"int"},
+ | {"name":"ghost","type":"int"}
+ |]}""".stripMargin)
+ val catalyst = new StructType().add("id", IntegerType)
+ val helper = new AvroUtils.AvroSchemaHelper(
+ avroWithRequiredGhost, catalyst, Seq.empty, Seq.empty, false)
+ val ex = assertThrows(classOf[IncompatibleSchemaException],
+ () => helper.validateNoExtraRequiredAvroFields())
+ assertTrue(ex.getMessage.contains("Found field 'ghost'"))
+
+ // A nullable extra Avro field is not required, so it is tolerated.
+ val avroWithOptionalGhost = parse(
+ """{"type":"record","name":"r","fields":[
+ | {"name":"id","type":"int"},
+ | {"name":"ghost","type":["null","int"]}
+ |]}""".stripMargin)
+ val helperWithOptionalGhost = new AvroUtils.AvroSchemaHelper(
+ avroWithOptionalGhost, catalyst, Seq.empty, Seq.empty, false)
+ assertDoesNotThrow(new Executable {
+ def execute(): Unit = helperWithOptionalGhost.validateNoExtraRequiredAvroFields()
+ })
+ }
+
+ @Test
+ def testGetFieldByNameAmbiguousMatch(): Unit = {
+ // Two fields differing only by case collide under a case-insensitive resolver.
+ val avro = parse(
+ """{"type":"record","name":"r","fields":[
+ | {"name":"ID","type":"int"},
+ | {"name":"id","type":"int"}
+ |]}""".stripMargin)
+ val helper = new AvroUtils.AvroSchemaHelper(avro, new StructType(), Seq.empty, Seq.empty, false)
+ // The resolver comes from ambient SQLConf, so pin it instead of relying on the JVM default.
+ val conf = SQLConf.get
+ conf.setConfString("spark.sql.caseSensitive", "false")
+ try {
+ val ex = assertThrows(classOf[IncompatibleSchemaException],
+ () => helper.getFieldByName("id"))
+ assertTrue(ex.getMessage.contains("gave 2 matches"))
+ // Under a case-sensitive resolver the same lookup is unambiguous.
+ conf.setConfString("spark.sql.caseSensitive", "true")
+ assertEquals("id", helper.getFieldByName("id").get.name())
+ } finally {
+ conf.unsetConf("spark.sql.caseSensitive")
+ }
+ }
+}