You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "lamber-ken (Jira)" <ji...@apache.org> on 2020/05/11 01:30:00 UTC
[jira] [Assigned] (HUDI-600) Cleaner fails with AVRO exception when
upgrading from 0.5.0 to master
[ https://issues.apache.org/jira/browse/HUDI-600?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
lamber-ken reassigned HUDI-600:
-------------------------------
Assignee: Balaji Varadarajan
> Cleaner fails with AVRO exception when upgrading from 0.5.0 to master
> ---------------------------------------------------------------------
>
> Key: HUDI-600
> URL: https://issues.apache.org/jira/browse/HUDI-600
> Project: Apache Hudi (incubating)
> Issue Type: Bug
> Components: Cleaner
> Reporter: Nishith Agarwal
> Assignee: Balaji Varadarajan
> Priority: Major
> Labels: help-requested
> Fix For: 0.6.0
>
>
> ```
> org.apache.avro.AvroTypeException: Found org.apache.hudi.avro.model.HoodieCleanMetadata, expecting org.apache.hudi.avro.model.HoodieCleanerPlan, missing required field policy
> at org.apache.avro.io.ResolvingDecoder.doAction(ResolvingDecoder.java:292)
> at org.apache.avro.io.parsing.Parser.advance(Parser.java:88)
> at org.apache.avro.io.ResolvingDecoder.readFieldOrder(ResolvingDecoder.java:130)
> at org.apache.avro.generic.GenericDatumReader.readRecord(GenericDatumReader.java:215)
> at org.apache.avro.generic.GenericDatumReader.readWithoutConversion(GenericDatumReader.java:175)
> at org.apache.avro.generic.GenericDatumReader.read(GenericDatumReader.java:153)
> at org.apache.avro.generic.GenericDatumReader.read(GenericDatumReader.java:145)
> at org.apache.avro.file.DataFileStream.next(DataFileStream.java:233)
> at org.apache.avro.file.DataFileStream.next(DataFileStream.java:220)
> at org.apache.hudi.common.util.AvroUtils.deserializeAvroMetadata(AvroUtils.java:149)
> at org.apache.hudi.common.util.CleanerUtils.getCleanerPlan(CleanerUtils.java:88)
> at org.apache.hudi.HoodieCleanClient.runClean(HoodieCleanClient.java:144)
> at org.apache.hudi.HoodieCleanClient.lambda$clean$0(HoodieCleanClient.java:89)
> at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
> at java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:647)
> at org.apache.hudi.HoodieCleanClient.clean(HoodieCleanClient.java:87)
> at org.apache.hudi.HoodieWriteClient.clean(HoodieWriteClient.java:837)
> at org.apache.hudi.HoodieWriteClient.postCommit(HoodieWriteClient.java:514)
> at org.apache.hudi.AbstractHoodieWriteClient.commit(AbstractHoodieWriteClient.java:156)
> at org.apache.hudi.AbstractHoodieWriteClient.commit(AbstractHoodieWriteClient.java:100)
> at org.apache.hudi.AbstractHoodieWriteClient.commit(AbstractHoodieWriteClient.java:91)
> at org.apache.hudi.HoodieSparkSqlWriter$.checkWriteStatus(HoodieSparkSqlWriter.scala:261)
> at org.apache.hudi.HoodieSparkSqlWriter$.write(HoodieSparkSqlWriter.scala:183)
> at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:91)
> at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:45)
> at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70)
> at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68)
> at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:86)
> at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131)
> at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127)
> at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:156)
> at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152)
> at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127)
> at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:80)
> at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:80)
> at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:676)
> at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:676)
> at org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:78)
> at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:125)
> at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:73)
> at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:676)
> at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:285)
> at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:271)
> at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:229)
> ```
>
> [~varadarb] any ideas about this ?
>
> [~thesquelched] fyi
--
This message was sent by Atlassian Jira
(v8.3.4#803005)