You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "voonhous (via GitHub)" <gi...@apache.org> on 2026/05/01 06:33:22 UTC
Re: [I] Ensure Variant type support on Spark 4.1 [hudi]
voonhous commented on issue #18605:
URL: https://github.com/apache/hudi/issues/18605#issuecomment-4358165856
Did not encounter a SIGSEV error, got this error instead:
```log
Illegal initial character: 0
org.apache.hudi.internal.schema.HoodieSchemaException: Illegal initial character: 0
at org.apache.hudi.HoodieSchemaConversionUtils$.convertStructTypeToHoodieSchema(HoodieSchemaConversionUtils.scala:143)
at org.apache.hudi.HoodieSchemaConversionUtils$.convertStructTypeToHoodieSchema(HoodieSchemaConversionUtils.scala:122)
at org.apache.hudi.HoodieSchemaConversionUtils$.convertStructTypeToHoodieSchema(HoodieSchemaConversionUtils.scala:107)
at org.apache.spark.sql.execution.datasources.parquet.HoodieFileGroupReaderBasedFileFormat.buildReaderWithPartitionValues(HoodieFileGroupReaderBasedFileFormat.scala:255)
at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD$lzycompute(DataSourceScanExec.scala:728)
at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD(DataSourceScanExec.scala:717)
at org.apache.spark.sql.execution.FileSourceScanExec.doExecuteColumnar(DataSourceScanExec.scala:767)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
at org.apache.spark.sql.execution.InputAdapter.doExecuteColumnar(WholeStageCodegenExec.scala:527)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
at org.apache.spark.sql.execution.ColumnarToRowExec.inputRDDs(Columnar.scala:205)
at org.apache.spark.sql.execution.aggregate.AggregateCodegenSupport.inputRDDs(AggregateCodegenSupport.scala:91)
at org.apache.spark.sql.execution.aggregate.AggregateCodegenSupport.inputRDDs$(AggregateCodegenSupport.scala:90)
at org.apache.spark.sql.execution.aggregate.HashAggregateExec.inputRDDs(HashAggregateExec.scala:51)
at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:763)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
at org.apache.spark.util.Utils$.getTryWithCallerStacktrace(Utils.scala:1453)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD$lzycompute(ShuffleExchangeExec.scala:209)
at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD(ShuffleExchangeExec.scala:209)
at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.mapOutputStatisticsFuture$lzycompute(ShuffleExchangeExec.scala:214)
at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.mapOutputStatisticsFuture(ShuffleExchangeExec.scala:213)
at org.apache.spark.sql.execution.exchange.ShuffleExchangeLike.$anonfun$triggerFuture$3(ShuffleExchangeExec.scala:97)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.exchange.ShuffleExchangeLike.$anonfun$triggerFuture$1(ShuffleExchangeExec.scala:97)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$4(SQLExecution.scala:335)
at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:285)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$3(SQLExecution.scala:333)
at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$2(SQLExecution.scala:329)
at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1768)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.spark.util.Utils$OriginalTryStackTraceException: Full stacktrace of original doTryWithCallerStacktrace caller
at org.apache.hudi.HoodieSchemaConversionUtils$.convertStructTypeToHoodieSchema(HoodieSchemaConversionUtils.scala:143)
at org.apache.hudi.HoodieSchemaConversionUtils$.convertStructTypeToHoodieSchema(HoodieSchemaConversionUtils.scala:122)
at org.apache.hudi.HoodieSchemaConversionUtils$.convertStructTypeToHoodieSchema(HoodieSchemaConversionUtils.scala:107)
at org.apache.spark.sql.execution.datasources.parquet.HoodieFileGroupReaderBasedFileFormat.buildReaderWithPartitionValues(HoodieFileGroupReaderBasedFileFormat.scala:255)
at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD$lzycompute(DataSourceScanExec.scala:728)
at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD(DataSourceScanExec.scala:717)
at org.apache.spark.sql.execution.FileSourceScanExec.doExecuteColumnar(DataSourceScanExec.scala:767)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
at org.apache.spark.sql.execution.InputAdapter.doExecuteColumnar(WholeStageCodegenExec.scala:527)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
at org.apache.spark.sql.execution.ColumnarToRowExec.inputRDDs(Columnar.scala:205)
at org.apache.spark.sql.execution.aggregate.AggregateCodegenSupport.inputRDDs(AggregateCodegenSupport.scala:91)
at org.apache.spark.sql.execution.aggregate.AggregateCodegenSupport.inputRDDs$(AggregateCodegenSupport.scala:90)
at org.apache.spark.sql.execution.aggregate.HashAggregateExec.inputRDDs(HashAggregateExec.scala:51)
at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:763)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
... 22 more
Caused by: org.apache.avro.SchemaParseException: Illegal initial character: 0
at org.apache.avro.Schema.validateName(Schema.java:1694)
at org.apache.avro.Schema$Field.<init>(Schema.java:576)
at org.apache.avro.Schema$Field.<init>(Schema.java:625)
at org.apache.hudi.avro.HoodieAvroUtils.createNewSchemaField(HoodieAvroUtils.java:288)
at org.apache.hudi.common.schema.HoodieSchemaField.of(HoodieSchemaField.java:122)
at org.apache.hudi.common.schema.HoodieSchemaField.of(HoodieSchemaField.java:98)
at org.apache.spark.sql.avro.HoodieSparkSchemaConverters$.$anonfun$toHoodieTypeNested$2(HoodieSparkSchemaConverters.scala:219)
at scala.collection.Iterator$$anon$9.next(Iterator.scala:594)
at scala.collection.immutable.List.prependedAll(List.scala:153)
at scala.collection.immutable.List$.from(List.scala:674)
at scala.collection.immutable.List$.from(List.scala:671)
at scala.collection.SeqFactory$Delegate.from(Factory.scala:306)
at scala.collection.immutable.Seq$.from(Seq.scala:42)
at scala.collection.immutable.Seq$.from(Seq.scala:39)
at scala.collection.IterableOps.map(Iterable.scala:683)
at scala.collection.IterableOps.map$(Iterable.scala:683)
at org.apache.spark.sql.types.StructType.map(StructType.scala:105)
at org.apache.spark.sql.avro.HoodieSparkSchemaConverters$.toHoodieTypeNested(HoodieSparkSchemaConverters.scala:209)
at org.apache.spark.sql.avro.HoodieSparkSchemaConverters$.$anonfun$toHoodieTypeNested$2(HoodieSparkSchemaConverters.scala:210)
at scala.collection.Iterator$$anon$9.next(Iterator.scala:594)
at scala.collection.immutable.List.prependedAll(List.scala:156)
at scala.collection.immutable.List$.from(List.scala:674)
at scala.collection.immutable.List$.from(List.scala:671)
at scala.collection.SeqFactory$Delegate.from(Factory.scala:306)
at scala.collection.immutable.Seq$.from(Seq.scala:42)
at scala.collection.immutable.Seq$.from(Seq.scala:39)
at scala.collection.IterableOps.map(Iterable.scala:683)
at scala.collection.IterableOps.map$(Iterable.scala:683)
at org.apache.spark.sql.types.StructType.map(StructType.scala:105)
at org.apache.spark.sql.avro.HoodieSparkSchemaConverters$.toHoodieTypeNested(HoodieSparkSchemaConverters.scala:209)
at org.apache.spark.sql.avro.HoodieSparkSchemaConverters$.toHoodieType(HoodieSparkSchemaConverters.scala:66)
at org.apache.hudi.HoodieSchemaConversionUtils$.convertStructTypeToHoodieSchema(HoodieSchemaConversionUtils.scala:140)
at org.apache.hudi.HoodieSchemaConversionUtils$.convertStructTypeToHoodieSchema(HoodieSchemaConversionUtils.scala:122)
at org.apache.hudi.HoodieSchemaConversionUtils$.convertStructTypeToHoodieSchema(HoodieSchemaConversionUtils.scala:107)
at org.apache.spark.sql.execution.datasources.parquet.HoodieFileGroupReaderBasedFileFormat.buildReaderWithPartitionValues(HoodieFileGroupReaderBasedFileFormat.scala:255)
at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD$lzycompute(DataSourceScanExec.scala:728)
at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD(DataSourceScanExec.scala:717)
at org.apache.spark.sql.execution.FileSourceScanExec.doExecuteColumnar(DataSourceScanExec.scala:767)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
at org.apache.spark.sql.execution.InputAdapter.doExecuteColumnar(WholeStageCodegenExec.scala:527)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
at org.apache.spark.sql.execution.ColumnarToRowExec.inputRDDs(Columnar.scala:205)
at org.apache.spark.sql.execution.aggregate.AggregateCodegenSupport.inputRDDs(AggregateCodegenSupport.scala:91)
at org.apache.spark.sql.execution.aggregate.AggregateCodegenSupport.inputRDDs$(AggregateCodegenSupport.scala:90)
at org.apache.spark.sql.execution.aggregate.HashAggregateExec.inputRDDs(HashAggregateExec.scala:51)
at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:763)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@hudi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org