You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "Filipe Souza (Jira)" <ji...@apache.org> on 2022/11/14 14:07:00 UTC
[jira] [Comment Edited] (HUDI-3393) Deltastreamer fails during replace commit rollback and cannot proceed
[ https://issues.apache.org/jira/browse/HUDI-3393?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17633820#comment-17633820 ]
Filipe Souza edited comment on HUDI-3393 at 11/14/22 2:06 PM:
--------------------------------------------------------------
I'm still having this issue in version 0.12.1.
Can you help me?
{code:java}
org.apache.hudi.exception.HoodieRollbackException: Failed to rollback vivo_recarga_current commits 20221111220422669
at org.apache.hudi.client.BaseHoodieWriteClient.rollback(BaseHoodieWriteClient.java:785) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.client.BaseHoodieWriteClient.rollbackFailedWrites(BaseHoodieWriteClient.java:1194) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.client.BaseHoodieWriteClient.rollbackFailedWrites(BaseHoodieWriteClient.java:1177) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.client.BaseHoodieWriteClient.rollbackFailedWrites(BaseHoodieWriteClient.java:1165) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.client.BaseHoodieWriteClient.lambda$startCommitWithTime$97cdbdca$1(BaseHoodieWriteClient.java:966) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.common.util.CleanerUtils.rollbackFailedWrites(CleanerUtils.java:153) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.client.BaseHoodieWriteClient.startCommitWithTime(BaseHoodieWriteClient.java:965) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.client.BaseHoodieWriteClient.startCommitWithTime(BaseHoodieWriteClient.java:958) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.HoodieSparkSqlWriter$.write(HoodieSparkSqlWriter.scala:330) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.spark.sql.hudi.command.InsertIntoHoodieTableCommand$.run(InsertIntoHoodieTableCommand.scala:101) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.spark.sql.hudi.command.InsertIntoHoodieTableCommand.run(InsertIntoHoodieTableCommand.scala:60) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:75) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:73) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:84) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:103) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:107) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:224) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.executeQuery$1(SQLExecution.scala:114) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$7(SQLExecution.scala:139) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:107) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:224) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$6(SQLExecution.scala:139) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:245) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:138) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:779) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:68) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:100) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:96) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:615) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:177) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:615) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:30) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:267) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:263) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:591) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:96) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:83) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:81) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.Dataset.<init>(Dataset.scala:222) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:102) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:779) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:99) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:622) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:779) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:617) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:651) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver.run(SparkSQLDriver.scala:67) ~[spark-hive-thriftserver_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processCmd(SparkSQLCLIDriver.scala:384) ~[spark-hive-thriftserver_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.$anonfun$processLine$1(SparkSQLCLIDriver.scala:504) ~[spark-hive-thriftserver_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.$anonfun$processLine$1$adapted(SparkSQLCLIDriver.scala:498) ~[spark-hive-thriftserver_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at scala.collection.Iterator.foreach(Iterator.scala:943) ~[scala-library-2.12.15.jar:?]
at scala.collection.Iterator.foreach$(Iterator.scala:943) ~[scala-library-2.12.15.jar:?]
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431) ~[scala-library-2.12.15.jar:?]
at scala.collection.IterableLike.foreach(IterableLike.scala:74) ~[scala-library-2.12.15.jar:?]
at scala.collection.IterableLike.foreach$(IterableLike.scala:73) ~[scala-library-2.12.15.jar:?]
at scala.collection.AbstractIterable.foreach(Iterable.scala:56) ~[scala-library-2.12.15.jar:?]
at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processLine(SparkSQLCLIDriver.scala:498) ~[spark-hive-thriftserver_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:336) ~[hive-cli-2.3.9-amzn-2.jar:2.3.9-amzn-2]
at org.apache.hadoop.hive.cli.CliDriver.processReader(CliDriver.java:474) ~[hive-cli-2.3.9-amzn-2.jar:2.3.9-amzn-2]
at org.apache.hadoop.hive.cli.CliDriver.processFile(CliDriver.java:490) ~[hive-cli-2.3.9-amzn-2.jar:2.3.9-amzn-2]
at org.apache.hadoop.hive.cli.CliDriver.processInitFiles(CliDriver.java:501) ~[hive-cli-2.3.9-amzn-2.jar:2.3.9-amzn-2]
at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver$.main(SparkSQLCLIDriver.scala:195) ~[spark-hive-thriftserver_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.main(SparkSQLCLIDriver.scala) ~[spark-hive-thriftserver_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_352]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_352]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_352]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_352]
at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1006) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1095) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1104) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
Caused by: org.apache.hudi.exception.HoodieRollbackException: Found commits after time :20221111220422669, please rollback greater commits first
at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.validateRollbackCommitSequence(BaseRollbackActionExecutor.java:171) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.doRollbackAndGetStats(BaseRollbackActionExecutor.java:210) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.runRollback(BaseRollbackActionExecutor.java:108) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.execute(BaseRollbackActionExecutor.java:135) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.table.HoodieSparkCopyOnWriteTable.rollback(HoodieSparkCopyOnWriteTable.java:281) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.client.BaseHoodieWriteClient.rollback(BaseHoodieWriteClient.java:768) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
... 76 more {code}
was (Author: JIRAUSER295756):
I'm still having this issue on version 0.12.1.
Can you help me?
{code:java}
org.apache.hudi.exception.HoodieRollbackException: Failed to rollback vivo_recarga_current commits 20221111220422669
at org.apache.hudi.client.BaseHoodieWriteClient.rollback(BaseHoodieWriteClient.java:785) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.client.BaseHoodieWriteClient.rollbackFailedWrites(BaseHoodieWriteClient.java:1194) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.client.BaseHoodieWriteClient.rollbackFailedWrites(BaseHoodieWriteClient.java:1177) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.client.BaseHoodieWriteClient.rollbackFailedWrites(BaseHoodieWriteClient.java:1165) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.client.BaseHoodieWriteClient.lambda$startCommitWithTime$97cdbdca$1(BaseHoodieWriteClient.java:966) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.common.util.CleanerUtils.rollbackFailedWrites(CleanerUtils.java:153) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.client.BaseHoodieWriteClient.startCommitWithTime(BaseHoodieWriteClient.java:965) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.client.BaseHoodieWriteClient.startCommitWithTime(BaseHoodieWriteClient.java:958) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.HoodieSparkSqlWriter$.write(HoodieSparkSqlWriter.scala:330) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.spark.sql.hudi.command.InsertIntoHoodieTableCommand$.run(InsertIntoHoodieTableCommand.scala:101) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.spark.sql.hudi.command.InsertIntoHoodieTableCommand.run(InsertIntoHoodieTableCommand.scala:60) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:75) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:73) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:84) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:103) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:107) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:224) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.executeQuery$1(SQLExecution.scala:114) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$7(SQLExecution.scala:139) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:107) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:224) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$6(SQLExecution.scala:139) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:245) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:138) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:779) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:68) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:100) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:96) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:615) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:177) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:615) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:30) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:267) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:263) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:30) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:591) ~[spark-catalyst_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:96) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:83) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:81) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.Dataset.<init>(Dataset.scala:222) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:102) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:779) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:99) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:622) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:779) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:617) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:651) ~[spark-sql_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver.run(SparkSQLDriver.scala:67) ~[spark-hive-thriftserver_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processCmd(SparkSQLCLIDriver.scala:384) ~[spark-hive-thriftserver_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.$anonfun$processLine$1(SparkSQLCLIDriver.scala:504) ~[spark-hive-thriftserver_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.$anonfun$processLine$1$adapted(SparkSQLCLIDriver.scala:498) ~[spark-hive-thriftserver_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at scala.collection.Iterator.foreach(Iterator.scala:943) ~[scala-library-2.12.15.jar:?]
at scala.collection.Iterator.foreach$(Iterator.scala:943) ~[scala-library-2.12.15.jar:?]
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431) ~[scala-library-2.12.15.jar:?]
at scala.collection.IterableLike.foreach(IterableLike.scala:74) ~[scala-library-2.12.15.jar:?]
at scala.collection.IterableLike.foreach$(IterableLike.scala:73) ~[scala-library-2.12.15.jar:?]
at scala.collection.AbstractIterable.foreach(Iterable.scala:56) ~[scala-library-2.12.15.jar:?]
at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processLine(SparkSQLCLIDriver.scala:498) ~[spark-hive-thriftserver_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:336) ~[hive-cli-2.3.9-amzn-2.jar:2.3.9-amzn-2]
at org.apache.hadoop.hive.cli.CliDriver.processReader(CliDriver.java:474) ~[hive-cli-2.3.9-amzn-2.jar:2.3.9-amzn-2]
at org.apache.hadoop.hive.cli.CliDriver.processFile(CliDriver.java:490) ~[hive-cli-2.3.9-amzn-2.jar:2.3.9-amzn-2]
at org.apache.hadoop.hive.cli.CliDriver.processInitFiles(CliDriver.java:501) ~[hive-cli-2.3.9-amzn-2.jar:2.3.9-amzn-2]
at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver$.main(SparkSQLCLIDriver.scala:195) ~[spark-hive-thriftserver_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.main(SparkSQLCLIDriver.scala) ~[spark-hive-thriftserver_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_352]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_352]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_352]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_352]
at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1006) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1095) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1104) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) ~[spark-core_2.12-3.3.0-amzn-0.jar:3.3.0-amzn-0]
Caused by: org.apache.hudi.exception.HoodieRollbackException: Found commits after time :20221111220422669, please rollback greater commits first
at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.validateRollbackCommitSequence(BaseRollbackActionExecutor.java:171) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.doRollbackAndGetStats(BaseRollbackActionExecutor.java:210) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.runRollback(BaseRollbackActionExecutor.java:108) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.execute(BaseRollbackActionExecutor.java:135) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.table.HoodieSparkCopyOnWriteTable.rollback(HoodieSparkCopyOnWriteTable.java:281) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
at org.apache.hudi.client.BaseHoodieWriteClient.rollback(BaseHoodieWriteClient.java:768) ~[hudi-spark3.3-bundle_2.12-0.12.1.jar:0.12.1]
... 76 more {code}
> Deltastreamer fails during replace commit rollback and cannot proceed
> ---------------------------------------------------------------------
>
> Key: HUDI-3393
> URL: https://issues.apache.org/jira/browse/HUDI-3393
> Project: Apache Hudi
> Issue Type: Task
> Components: deltastreamer
> Reporter: sivabalan narayanan
> Assignee: sivabalan narayanan
> Priority: Critical
> Fix For: 0.11.0
>
> Attachments: image.png
>
> Original Estimate: 3h
> Remaining Estimate: 3h
>
> When a clustering commit failed mid-way and deltastreamer restarted.
> there could be one delta commit that could have succeeded. And so next time when clustering was attempted, it tries to rollback the older pending clustering and fails with "found commits after X, rollback greater commits first".
>
> {code:java}
> [2022-02-08 09:01:55,808] {{ssh_operator.py:143}} INFO - 22/02/08 09:01:55 INFO Client: client token: N/A diagnostics: User class threw exception: org.apache.hudi.exception.HoodieRollbackException: Found commits after time :20220208081622, please rollback greater commits first at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.validateRollbackCommitSequence(BaseRollbackActionExecutor.java:148) at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.doRollbackAndGetStats(BaseRollbackActionExecutor.java:179) at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.execute(BaseRollbackActionExecutor.java:102) at org.apache.hudi.table.HoodieSparkCopyOnWriteTable.rollback(HoodieSparkCopyOnWriteTable.java:237) at org.apache.hudi.client.AbstractHoodieWriteClient.rollbackInflightClustering(AbstractHoodieWriteClient.java:971) at org.apache.hudi.client.SparkRDDWriteClient.cluster(SparkRDDWriteClient.java:341) at org.apache.hudi.client.AbstractHoodieWriteClient.lambda$runAnyPendingClustering$1(AbstractHoodieWriteClient.java:475) at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384) at java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:647) at org.apache.hudi.client.AbstractHoodieWriteClient.runAnyPendingClustering(AbstractHoodieWriteClient.java:471) at org.apache.hudi.client.AbstractHoodieWriteClient.runTableServicesInline(AbstractHoodieWriteClient.java:453) at org.apache.hudi.client.AbstractHoodieWriteClient.commitStats(AbstractHoodieWriteClient.java:194) at org.apache.hudi.client.SparkRDDWriteClient.commit(SparkRDDWriteClient.java:121) at org.apache.hudi.client.SparkRDDWriteClient.commit(SparkRDDWriteClient.java:72) at org.apache.hudi.client.AbstractHoodieWriteClient.commit(AbstractHoodieWriteClient.java:162) at org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:465) at org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:282) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.lambda$sync$2(HoodieDeltaStreamer.java:172) at org.apache.hudi.common.util.Option.ifPresent(Option.java:96) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.sync(HoodieDeltaStreamer.java:170) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.main(HoodieDeltaStreamer.java:472) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:735) ApplicationMaster host: ip-172-19-25-155.eu-west-1.compute.internal ApplicationMaster RPC port: 38211 queue: default start time: 1644310580365 final status: FAILED tracking URL: http://ip-172-19-25-101.eu-west-1.compute.internal:20888/proxy/application_1643535866994_31483/ user: root22/02/08 09:01:55 ERROR Client: Application diagnostics message: User class threw exception: org.apache.hudi.exception.HoodieRollbackException: Found commits after time :20220208081622, please rollback greater commits first at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.validateRollbackCommitSequence(BaseRollbackActionExecutor.java:148) at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.doRollbackAndGetStats(BaseRollbackActionExecutor.java:179) at org.apache.hudi.table.action.rollback.BaseRollbackActionExecutor.execute(BaseRollbackActionExecutor.java:102) at org.apache.hudi.table.HoodieSparkCopyOnWriteTable.rollback(HoodieSparkCopyOnWriteTable.java:237) at org.apache.hudi.client.AbstractHoodieWriteClient.rollbackInflightClustering(AbstractHoodieWriteClient.java:971) at org.apache.hudi.client.SparkRDDWriteClient.cluster(SparkRDDWriteClient.java:341) at org.apache.hudi.client.AbstractHoodieWriteClient.lambda$runAnyPendingClustering$1(AbstractHoodieWriteClient.java:475) at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384) at java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:647) at org.apache.hudi.client.AbstractHoodieWriteClient.runAnyPendingClustering(AbstractHoodieWriteClient.java:471) at org.apache.hudi.client.AbstractHoodieWriteClient.runTableServicesInline(AbstractHoodieWriteClient.java:453) at org.apache.hudi.client.AbstractHoodieWriteClient.commitStats(AbstractHoodieWriteClient.java:194) at org.apache.hudi.client.SparkRDDWriteClient.commit(SparkRDDWriteClient.java:121) at org.apache.hudi.client.SparkRDDWriteClient.commit(SparkRDDWriteClient.java:72) at org.apache.hudi.client.AbstractHoodieWriteClient.commit(AbstractHoodieWriteClient.java:162) at org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:465) at org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:282) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.lambda$sync$2(HoodieDeltaStreamer.java:172) at org.apache.hudi.common.util.Option.ifPresent(Option.java:96) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.sync(HoodieDeltaStreamer.java:170) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.main(HoodieDeltaStreamer.java:472) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:735) {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)