You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "Ethan Guo (Jira)" <ji...@apache.org> on 2024/03/29 01:21:00 UTC

[jira] [Updated] (HUDI-3641) Timeline server have different last known instant in the timeline from client

     [ https://issues.apache.org/jira/browse/HUDI-3641?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]

Ethan Guo updated HUDI-3641:
----------------------------
    Component/s: cleaning
                 compaction
                 table-service

> Timeline server have different last known instant in the timeline from client
> -----------------------------------------------------------------------------
>
>                 Key: HUDI-3641
>                 URL: https://issues.apache.org/jira/browse/HUDI-3641
>             Project: Apache Hudi
>          Issue Type: Bug
>          Components: cleaning, compaction, table-service
>            Reporter: Ethan Guo
>            Priority: Major
>
> Hudi 0.9.0 Multi-writer scenario, one writer doing ingesting with Deltastreamer continuous mode, MOR, inserts, async compaction and cleaning (partitions under 2022/1, 2022/2), another writer with Spark datasource doing backfills to different partitions (2021/12).  At some point, the backfill job hits the following exception, due to timeline server having different last known instant in the timeline from client. We need triage if this still happens in latest master.
> {code:java}
> 22/03/14 12:23:49 ERROR RequestHandler: Got runtime exception servicing request partition=2021%2F12%2F1&maxinstant=20220314122055&includependingcompaction=false&basepath=file%3A%2FUsers%2Fethan%2FWork%2Fscripts%2Fmt_rollout_testing%2Fdeploy_c_multi_writer%2Fc5_mor_09mt_011mt%2Ftest_table&lastinstantts=20220314122055&timelinehash=9fa9326a9f7d25818e648df109d89ef5ddde53021dbc5b4bd234ff8e0d0429e8
> java.lang.IllegalArgumentException: Last known instant from client was 20220314122055 but server has the following timeline [[20220314122055__deltacommit__COMPLETED], [20220314122143__deltacommit__COMPLETED]]
>     at org.apache.hudi.common.util.ValidationUtils.checkArgument(ValidationUtils.java:40)
>     at org.apache.hudi.timeline.service.RequestHandler$ViewHandler.handle(RequestHandler.java:510)
>     at io.javalin.security.SecurityUtil.noopAccessManager(SecurityUtil.kt:22)
>     at io.javalin.Javalin.lambda$addHandler$0(Javalin.java:606)
>     at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:46)
>     at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:17)
>     at io.javalin.core.JavalinServlet$service$1.invoke(JavalinServlet.kt:143)
>     at io.javalin.core.JavalinServlet$service$2.invoke(JavalinServlet.kt:41)
>     at io.javalin.core.JavalinServlet.service(JavalinServlet.kt:107)
>     at io.javalin.core.util.JettyServerUtil$initialize$httpHandler$1.doHandle(JettyServerUtil.kt:72)
>     at org.apache.hudi.org.apache.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
>     at org.apache.hudi.org.apache.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
>     at org.apache.hudi.org.apache.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1668)
>     at org.apache.hudi.org.apache.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
>     at org.apache.hudi.org.apache.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
>     at org.apache.hudi.org.apache.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
>     at org.apache.hudi.org.apache.jetty.server.handler.HandlerList.handle(HandlerList.java:61)
>     at org.apache.hudi.org.apache.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:174)
>     at org.apache.hudi.org.apache.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
>     at org.apache.hudi.org.apache.jetty.server.Server.handle(Server.java:502)
>     at org.apache.hudi.org.apache.jetty.server.HttpChannel.handle(HttpChannel.java:370)
>     at org.apache.hudi.org.apache.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
>     at org.apache.hudi.org.apache.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
>     at org.apache.hudi.org.apache.jetty.io.FillInterest.fillable(FillInterest.java:103)
>     at org.apache.hudi.org.apache.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
>     at org.apache.hudi.org.apache.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
>     at org.apache.hudi.org.apache.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
>     at java.lang.Thread.run(Thread.java:748)
> 22/03/14 12:23:49 WARN ExceptionMapper: Uncaught exception
> java.lang.IllegalArgumentException: Last known instant from client was 20220314122055 but server has the following timeline [[20220314122055__deltacommit__COMPLETED], [20220314122143__deltacommit__COMPLETED]]
>     at org.apache.hudi.common.util.ValidationUtils.checkArgument(ValidationUtils.java:40)
>     at org.apache.hudi.timeline.service.RequestHandler$ViewHandler.handle(RequestHandler.java:510)
>     at io.javalin.security.SecurityUtil.noopAccessManager(SecurityUtil.kt:22)
>     at io.javalin.Javalin.lambda$addHandler$0(Javalin.java:606)
>     at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:46)
>     at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:17)
>     at io.javalin.core.JavalinServlet$service$1.invoke(JavalinServlet.kt:143)
>     at io.javalin.core.JavalinServlet$service$2.invoke(JavalinServlet.kt:41)
>     at io.javalin.core.JavalinServlet.service(JavalinServlet.kt:107)
>     at io.javalin.core.util.JettyServerUtil$initialize$httpHandler$1.doHandle(JettyServerUtil.kt:72)
>     at org.apache.hudi.org.apache.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
>     at org.apache.hudi.org.apache.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
>     at org.apache.hudi.org.apache.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1668)
>     at org.apache.hudi.org.apache.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
>     at org.apache.hudi.org.apache.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
>     at org.apache.hudi.org.apache.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
>     at org.apache.hudi.org.apache.jetty.server.handler.HandlerList.handle(HandlerList.java:61)
>     at org.apache.hudi.org.apache.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:174)
>     at org.apache.hudi.org.apache.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
>     at org.apache.hudi.org.apache.jetty.server.Server.handle(Server.java:502)
>     at org.apache.hudi.org.apache.jetty.server.HttpChannel.handle(HttpChannel.java:370)
>     at org.apache.hudi.org.apache.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
>     at org.apache.hudi.org.apache.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
>     at org.apache.hudi.org.apache.jetty.io.FillInterest.fillable(FillInterest.java:103)
>     at org.apache.hudi.org.apache.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
>     at org.apache.hudi.org.apache.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
>     at org.apache.hudi.org.apache.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
>     at java.lang.Thread.run(Thread.java:748)
> 22/03/14 12:23:49 ERROR PriorityBasedFileSystemView: Got error running preferred function. Trying secondary
> org.apache.hudi.exception.HoodieRemoteException: Server Error
>     at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.getLatestFileSlicesBeforeOrOn(RemoteHoodieTableFileSystemView.java:324)
>     at org.apache.hudi.common.table.view.PriorityBasedFileSystemView.execute(PriorityBasedFileSystemView.java:113)
>     at org.apache.hudi.common.table.view.PriorityBasedFileSystemView.getLatestFileSlicesBeforeOrOn(PriorityBasedFileSystemView.java:172)
>     at org.apache.hudi.table.action.deltacommit.SparkUpsertDeltaCommitPartitioner.getSmallFiles(SparkUpsertDeltaCommitPartitioner.java:72)
>     at org.apache.hudi.table.action.commit.UpsertPartitioner.lambda$getSmallFilesForPartitions$f1d92f9e$1(UpsertPartitioner.java:256)
>     at org.apache.spark.api.java.JavaPairRDD$.$anonfun$pairFunToScalaFun$1(JavaPairRDD.scala:1073)
>     at scala.collection.Iterator$$anon$10.next(Iterator.scala:459)
>     at scala.collection.Iterator.foreach(Iterator.scala:941)
>     at scala.collection.Iterator.foreach$(Iterator.scala:941)
>     at scala.collection.AbstractIterator.foreach(Iterator.scala:1429)
>     at scala.collection.generic.Growable.$plus$plus$eq(Growable.scala:62)
>     at scala.collection.generic.Growable.$plus$plus$eq$(Growable.scala:53)
>     at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:105)
>     at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:49)
>     at scala.collection.TraversableOnce.to(TraversableOnce.scala:315)
>     at scala.collection.TraversableOnce.to$(TraversableOnce.scala:313)
>     at scala.collection.AbstractIterator.to(Iterator.scala:1429)
>     at scala.collection.TraversableOnce.toBuffer(TraversableOnce.scala:307)
>     at scala.collection.TraversableOnce.toBuffer$(TraversableOnce.scala:307)
>     at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1429)
>     at scala.collection.TraversableOnce.toArray(TraversableOnce.scala:294)
>     at scala.collection.TraversableOnce.toArray$(TraversableOnce.scala:288)
>     at scala.collection.AbstractIterator.toArray(Iterator.scala:1429)
>     at org.apache.spark.rdd.RDD.$anonfun$collect$2(RDD.scala:1030)
>     at org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2236)
>     at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
>     at org.apache.spark.scheduler.Task.run(Task.scala:131)
>     at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
>     at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
>     at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
>     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
>     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
>     at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.http.client.HttpResponseException: Server Error
>     at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:70)
>     at org.apache.http.client.fluent.Response.handleResponse(Response.java:90)
>     at org.apache.http.client.fluent.Response.returnContent(Response.java:97)
>     at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:179)
>     at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.getLatestFileSlicesBeforeOrOn(RemoteHoodieTableFileSystemView.java:320)
>     ... 32 more {code}
>  
>  



--
This message was sent by Atlassian Jira
(v8.20.10#820010)