You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "Alexey Kudinkin (Jira)" <ji...@apache.org> on 2023/02/01 02:58:00 UTC
[jira] [Created] (HUDI-5670) Server-based Marker creation times out
Alexey Kudinkin created HUDI-5670:
-------------------------------------
Summary: Server-based Marker creation times out
Key: HUDI-5670
URL: https://issues.apache.org/jira/browse/HUDI-5670
Project: Apache Hudi
Issue Type: Bug
Components: writer-core
Reporter: Alexey Kudinkin
Assignee: Ethan Guo
Running writing benchmarks w/ SparkRecordMerger enabled hitting this SocketTimeoutException trying to create markers:
{code:java}
ception.HoodieRemoteException: Failed to create marker file partition=2020%2F10%2F29/69adadb4-d7ae-4b30-8af1-92ffa38be7df-0_1362-352-97811_20230201020238055.parquet.marker.CREATE
Read timed out
at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:121)
at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:46)
at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:486)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:492)
at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223)
at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352)
at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1535)
at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1445)
at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1509)
at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1332)
at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:376)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:327)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:136)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1504)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
Caused by: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieRemoteException: Failed to create marker file partition=2020%2F10%2F29/69adadb4-d7ae-4b30-8af1-92ffa38be7df-0_1362-352-97811_20230201020238055.parquet.marker.CREATE
Read timed out
at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:84)
at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:39)
at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:119)
... 22 more
Caused by: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieRemoteException: Failed to create marker file partition=2020%2F10%2F29/69adadb4-d7ae-4b30-8af1-92ffa38be7df-0_1362-352-97811_20230201020238055.parquet.marker.CREATE
Read timed out
at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:84) [3324/4704]
at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:39)
at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:119)
... 22 more
Caused by: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieRemoteException: Failed to create marker file partition=2020%2F11%2F29/8e3045e1-6de0-492e-bc34-85e2b8502767-0_1207-352-97656_20230201020238055.parquet.marker.CREATE
Read timed out
at org.apache.hudi.common.util.queue.SimpleExecutor.execute(SimpleExecutor.java:73)
at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:80)
... 24 more
Caused by: org.apache.hudi.exception.HoodieRemoteException: Failed to create marker file partition=2020%2F11%2F29/8e3045e1-6de0-492e-bc34-85e2b8502767-0_1207-352-97656_20230201020238055.parquet.marker.CREATE
Read timed out
at org.apache.hudi.table.marker.TimelineServerBasedWriteMarkers.executeCreateMarkerRequest(TimelineServerBasedWriteMarkers.java:186)
at org.apache.hudi.table.marker.TimelineServerBasedWriteMarkers.create(TimelineServerBasedWriteMarkers.java:141)
at org.apache.hudi.table.marker.WriteMarkers.create(WriteMarkers.java:95)
at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:139)
at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:102)
at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:75)
at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:45)
at org.apache.hudi.execution.CopyOnWriteInsertHandler.consume(CopyOnWriteInsertHandler.java:85)
at org.apache.hudi.execution.CopyOnWriteInsertHandler.consume(CopyOnWriteInsertHandler.java:42)
at org.apache.hudi.common.util.queue.SimpleExecutor.execute(SimpleExecutor.java:67)
... 25 more
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.hudi.org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
at org.apache.hudi.org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
at org.apache.hudi.org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
at org.apache.hudi.org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
at org.apache.hudi.org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
at org.apache.hudi.org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
at org.apache.hudi.org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
at org.apache.hudi.org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
at org.apache.hudi.org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
at org.apache.hudi.org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
at org.apache.hudi.org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
at org.apache.hudi.org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
at org.apache.hudi.org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
at org.apache.hudi.org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.hudi.org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
at org.apache.hudi.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
at org.apache.hudi.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
at org.apache.hudi.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
at org.apache.hudi.org.apache.http.client.fluent.Request.execute(Request.java:151)
at org.apache.hudi.table.marker.TimelineServerBasedWriteMarkers.executeRequestToTimelineServer(TimelineServerBasedWriteMarkers.java:232)
at org.apache.hudi.table.marker.TimelineServerBasedWriteMarkers.executeCreateMarkerRequest(TimelineServerBasedWriteMarkers.java:182)
... 34 more {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)