You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by "Kaijie Chen (Jira)" <ji...@apache.org> on 2022/04/26 02:48:00 UTC
[jira] [Updated] (HDDS-6645) Intermittent failure in TestOzoneFileSystem#testTrash
[ https://issues.apache.org/jira/browse/HDDS-6645?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Kaijie Chen updated HDDS-6645:
------------------------------
Description:
[https://github.com/kaijchen/ozone/runs/6150521370?check_suite_focus=true#step:4:2934]
{code:java}
Error: Tests run: 136, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: 416.797 s <<< FAILURE! - in org.apache.hadoop.fs.ozone.TestOzoneFileSystem
2935Error: TestOzoneFileSystem.testTrash Time elapsed: 120.381 s <<< ERROR!
2936java.util.concurrent.TimeoutException:
2937Timed out waiting for condition. Thread diagnostics:
2938Timestamp: 2022-04-25 01:41:34,792
2939
2940"qtp1740509828-221-acceptor-0@6917bbb9-ServerConnector@6644d0cc{HTTP/1.1, (http/1.1)}{0.0.0.0:41183}" daemon prio=3 tid=221 runnable
2941java.lang.Thread.State: RUNNABLE
2942 at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
2943 at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421)
2944 at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249)
2945 at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
2946 at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
2947 at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
2948 at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
2949 at java.lang.Thread.run(Thread.java:750)
2950"IPC Server handler 8 on default port 41079" daemon prio=5 tid=61 timed_waiting
2951java.lang.Thread.State: TIMED_WAITING
2952 at sun.misc.Unsafe.park(Native Method)
2953 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
2954 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
2955 at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
2956 at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
2957 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2932)
2958"Datanode ReportManager Thread - 0" daemon prio=5 tid=205 in Object.wait()
2959java.lang.Thread.State: WAITING (on object monitor)
2960 at sun.misc.Unsafe.park(Native Method)
2961 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
2962 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
2963 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
2964 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
2965 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
2966 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
2967 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
2968 at java.lang.Thread.run(Thread.java:750)
2969"IPC Server handler 14 on default port 41079" daemon prio=5 tid=67 timed_waiting
2970java.lang.Thread.State: TIMED_WAITING
2971 at sun.misc.Unsafe.park(Native Method)
2972 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
2973 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
2974 at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
2975 at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
2976 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2932)
2977"BlockDeletingService#2" daemon prio=5 tid=659 in Object.wait()
2978java.lang.Thread.State: WAITING (on object monitor)
2979 at sun.misc.Unsafe.park(Native Method)
2980 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
2981 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
2982 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
2983 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
2984 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
2985 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
2986 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
2987 at java.lang.Thread.run(Thread.java:750)
2988"IPC Server handler 18 on default port 36721" daemon prio=5 tid=91 timed_waiting
2989java.lang.Thread.State: TIMED_WAITING
2990 at sun.misc.Unsafe.park(Native Method)
2991 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
2992 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
2993 at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
2994 at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
2995 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2932)
2996"BlockDeletingService#1" daemon prio=5 tid=308 in Object.wait()
2997java.lang.Thread.State: WAITING (on object monitor)
2998 at sun.misc.Unsafe.park(Native Method)
2999 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
3000 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
3001 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
3002 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
3003 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
3004 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
3005 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
3006 at java.lang.Thread.run(Thread.java:750)
3007"qtp1849335409-249" daemon prio=5 tid=249 timed_waiting
3008java.lang.Thread.State: TIMED_WAITING
3009 at sun.misc.Unsafe.park(Native Method)
3010 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
3011 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
3012 at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
3013 at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
3014 at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
3015 at java.lang.Thread.run(Thread.java:750)
3016"BlockDeletingService#0" daemon prio=5 tid=327 timed_waiting
3017java.lang.Thread.State: TIMED_WAITING
3018 at sun.misc.Unsafe.park(Native Method)
3019 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
3020 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
3021 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
3022 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
3023 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
3024 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
3025 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
3026 at java.lang.Thread.run(Thread.java:750)
3027"Datanode State Machine Task Thread - 1" prio=5 tid=319 in Object.wait()
3028java.lang.Thread.State: WAITING (on object monitor)
3029 at sun.misc.Unsafe.park(Native Method)
3030 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
3031 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
3032 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
3033 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
3034 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
3035 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
3036 at java.lang.Thread.run(Thread.java:750)
3037"BlockDeletingService#2" daemon prio=5 tid=656 in Object.wait()
3038java.lang.Thread.State: WAITING (on object monitor)
3039 at sun.misc.Unsafe.park(Native Method)
3040 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
3041 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
3042 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
3043 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
3044 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
3045 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
3046 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
3047 at java.lang.Thread.run(Thread.java:750)
3048"pool-55-thread-3" prio=5 tid=587 in Object.wait()
3049java.lang.Thread.State: WAITING (on object monitor)
3050 at sun.misc.Unsafe.park(Native Method)
3051 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
3052 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
3053 at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
3054 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
3055 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
3056 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
3057 at java.lang.Thread.run(Thread.java:750)
3058"qtp1377489126-173" daemon prio=5 tid=173 timed_waiting
3059java.lang.Thread.State: TIMED_WAITING
3060 at sun.misc.Unsafe.park(Native Method)
3061 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
3062 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
3063 at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
3064 at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
3065 at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
3066 at java.lang.Thread.run(Thread.java:750)
3067"IPC Server handler 4 on default port 40135" daemon prio=5 tid=142 timed_waiting
3068java.lang.Thread.State: TIMED_WAITING
3069 at sun.misc.Unsafe.park(Native Method)
3070 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
3071 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
3072 at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
3073 at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
3074 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2932)
3075"IPC Server listener on 0" daemon prio=5 tid=117 runnable
3076java.lang.Thread.State: RUNNABLE
3077 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
3078 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
3079 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
3080 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
3081 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
3082 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
3083 at org.apache.hadoop.ipc.Server$Listener.run(Server.java:1350)
3084"IPC Server handler 17 on default port 35237" daemon prio=5 tid=50 timed_waiting
3085java.lang.Thread.State: TIMED_WAITING
3086 at sun.misc.Unsafe.park(Native Method)
3087 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
3088 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
3089 at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
3090 at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
3091 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2932)
3092"EndpointStateMachine task thread for /0.0.0.0:36721 - 0 " prio=5 tid=330 in Object.wait()
3093java.lang.Thread.State: WAITING (on object monitor)
3094 at sun.misc.Unsafe.park(Native Method)
3095 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
3096 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
3097 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
3098 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
3099 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
3100 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
3101 at java.lang.Thread.run(Thread.java:750)
3102"BlockDeletingService#0" daemon prio=5 tid=317 timed_waiting
3103java.lang.Thread.State: TIMED_WAITING
3104 at sun.misc.Unsafe.park(Native Method)
3105 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
3106 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
3107 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
3108 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
3109 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
3110 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
3111 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
3112 at java.lang.Thread.run(Thread.java:750)
3113"qtp577672681-100" daemon prio=5 tid=100 timed_waiting
3114java.lang.Thread.State: TIMED_WAITING
3115 at sun.misc.Unsafe.park(Native Method)
3116 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
3117 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
3118 at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
3119 at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
3120 at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
3121 at java.lang.Thread.run(Thread.java:750)
3122"pool-152-thread-1" prio=5 tid=398 in Object.wait()
3123java.lang.Thread.State: WAITING (on object monitor)
3124 at sun.misc.Unsafe.park(Native Method)
3125 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
3126 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
3127 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
3128 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
3129 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
3130 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
3131 at java.lang.Thread.run(Thread.java:750)
3132"Datanode ReportManager Thread - 2" daemon prio=5 tid=180 in Object.wait()
3133java.lang.Thread.State: WAITING (on object monitor)
3134 at sun.misc.Unsafe.park(Native Method)
3135 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
3136 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
3137 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
3138 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
3139 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
3140 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
3141 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
3142 at java.lang.Thread.run(Thread.java:750)
3143"d3eea043-2c2b-4d78-81f5-cd3edfbb9ed9@group-24FF75A3605B-StateMachineUpdater" daemon prio=5 tid=364 in Object.wait()
3144java.lang.Thread.State: WAITING (on object monitor)
3145 at java.lang.Object.wait(Native Method)
3146 at java.lang.Object.wait(Object.java:502)
3147 at org.apache.ratis.server.impl.StateMachineUpdater.waitForCommit(StateMachineUpdater.java:204)
3148 at org.apache.ratis.server.impl.StateMachineUpdater.run(StateMachineUpdater.java:173)
3149 at java.lang.Thread.run(Thread.java:750)
3150"Datanode State Machine Task Thread - 1" prio=5 tid=288 in Object.wait()
3151java.lang.Thread.State: WAITING (on object monitor)
3152 at sun.misc.Unsafe.park(Native Method)
3153 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
3154 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
3155 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
3156 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
3157 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
3158 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
3159 at java.lang.Thread.run(Thread.java:750)
3160"pool-145-thread-1" daemon prio=5 tid=266 timed_waiting
3161java.lang.Thread.State: TIMED_WAITING
3162 at sun.misc.Unsafe.park(Native Method)
3163 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
3164 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
3165 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
3166 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
3167 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
3168 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
3169 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
3170 at java.lang.Thread.run(Thread.java:750)
3171"ContainerOp-b606fafc-cf14-4ded-8f6e-d56353c5b67d-6" prio=5 tid=523 in Object.wait()
3172java.lang.Thread.State: WAITING (on object monitor)
3173 at sun.misc.Unsafe.park(Native Method)
3174 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
3175 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
3176 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
3177 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
3178 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
3179 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
3180 at java.lang.Thread.run(Thread.java:750)
3181"surefire-forkedjvm-command-thread" daemon prio=5 tid=10 runnable
3182java.lang.Thread.State: RUNNABLE
3183 at java.io.FileInputStream.readBytes(Native Method)
3184 at java.io.FileInputStream.read(FileInputStream.java:255)
3185 at java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
3186 at java.io.BufferedInputStream.read(BufferedInputStream.java:345)
3187 at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
3188 at java.io.BufferedInputStream.read1(BufferedInputStream.java:286)
3189 at java.io.BufferedInputStream.read(BufferedInputStream.java:345)
3190 at org.apache.maven.surefire.api.util.internal.Channels$3.readImpl(Channels.java:214)
3191 at org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleReadableChannel.read(AbstractNoninterruptibleReadableChannel.java:54)
3192 at org.apache.maven.surefire.booter.spi.LegacyMasterProcessChannelDecoder.decode(LegacyMasterProcessChannelDecoder.java:80)
3193 at org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:343)
3194 at java.lang.Thread.run(Thread.java:750)
3195"0698bb1e-527a-4ce6-96df-4eb9049c94b6@group-24FF75A3605B->cadadf1f-4933-4fd8-9a68-0ae33772136e-GrpcLogAppender-LogAppenderDaemon" daemon prio=5 tid=579 timed_waiting
3196java.lang.Thread.State: TIMED_WAITING
3197 at java.lang.Object.wait(Native Method)
3198 at org.apache.ratis.grpc.server.GrpcLogAppender.mayWait(GrpcLogAppender.java:175)
3199 at org.apache.ratis.grpc.server.GrpcLogAppender.run(GrpcLogAppender.java:127)
3200 at org.apache.ratis.server.leader.LogAppenderDaemon.run(LogAppenderDaemon.java:77)
3201 at org.apache.ratis.server.leader.LogAppenderDaemon$$Lambda$859/1063038588.run(Unknown Source)
3202 at java.lang.Thread.run(Thread.java:750)
3203"d3eea043-2c2b-4d78-81f5-cd3edfbb9ed9@group-24FF75A3605B-FollowerState" daemon prio=5 tid=576 timed_waiting
3204java.lang.Thread.State: TIMED_WAITING
3205 at java.lang.Thread.sleep(Native Method)
3206 at java.lang.Thread.sleep(Thread.java:342)
3207 at java.util.concurrent.TimeUnit.sleep(TimeUnit.java:386)
3208 at org.apache.ratis.util.TimeDuration.sleep(TimeDuration.java:324)
3209 at org.apache.ratis.util.TimeDuration.sleep(TimeDuration.java:309)
3210 at org.apache.ratis.server.impl.FollowerState.run(FollowerState.java:118)
3211"ContainerOp-b606fafc-cf14-4ded-8f6e-d56353c5b67d-8" prio=5 tid=541 in Object.wait()
3212java.lang.Thread.State: WAITING (on object monitor)
3213 at sun.misc.Unsafe.park(Native Method)
3214 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
3215 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
3216 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
3217 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
3218 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
3219 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
3220 at java.lang.Thread.run(Thread.java:750)
3221"org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DeleteBlocksCommandHandler$DeleteCmdWorker@26dcac0d" daemon prio=5 tid=218 timed_waiting
3222java.lang.Thread.State: TIMED_WAITING
3223 at java.lang.Thread.sleep(Native Method)
3224 at org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DeleteBlocksCommandHandler$DeleteCmdWorker.run(DeleteBlocksCommandHandler.java:186)
3225 at java.lang.Thread.run(Thread.java:750)
3226"org.apache.hadoop.util.JvmPauseMonitor$Monitor@5b74c45a" daemon prio=5 tid=159 timed_waiting
3227java.lang.Thread.State: TIMED_WAITING
3228 at java.lang.Thread.sleep(Native Method)
3229 at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
3230 at java.lang.Thread.run(Thread.java:750)
3231"IPC Server handler 9 on default port 40135" daemon prio=5 tid=147 timed_waiting
3232java.lang.Thread.State: TIMED_WAITING
3233 at sun.misc.Unsafe.park(Native Method)
3234 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
3235 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
3236 at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
3237 at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
3238 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2932)
3239"IPC Server handler 3 on default port 41079" daemon prio=5 tid=56 timed_waiting
3240java.lang.Thread.State: TIMED_WAITING
3241 at sun.misc.Unsafe.park(Native Method)
3242 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
3243 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
3244 at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
3245 at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
3246 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2932)
3247"cadadf1f-4933-4fd8-9a68-0ae33772136e@group-C4594F0DA3B7-LeaderStateImpl" daemon prio=5 tid=408 timed_waiting
3248java.lang.Thread.State: TIMED_WAITING
3249 at sun.misc.Unsafe.park(Native Method)
3250 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
3251 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
3252 at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418)
3253 at org.apache.ratis.server.impl.LeaderStateImpl$EventQueue.poll(LeaderStateImpl.java:156)
3254 at org.apache.ratis.server.impl.LeaderStateImpl$EventProcessor.run(LeaderStateImpl.java:600)
3255"cadadf1f-4933-4fd8-9a68-0ae33772136e@group-24FF75A3605B-StateMachineUpdater" daemon prio=5 tid=354 in Object.wait()
3256java.lang.Thread.State: WAITING (on object monitor)
3257 at java.lang.Object.wait(Native Method)
3258 at java.lang.Object.wait(Object.java:502)
3259 at org.apache.ratis.server.impl.StateMachineUpdater.waitForCommit(StateMachineUpdater.java:204)
3260 at org.apache.ratis.server.impl.StateMachineUpdater.run(StateMachineUpdater.java:173)
3261 at java.lang.Thread.run(Thread.java:750)
3262"ChunkWriter-1-0" daemon prio=5 tid=322 in Object.wait()
3263java.lang.Thread.State: WAITING (on object monitor)
3264 at sun.misc.Unsafe.park(Native Method)
3265 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
3266 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
3267 at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
3268 at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
3269 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
3270 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
3271 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
3272 at java.lang.Thread.run(Thread.java:750)
3273"Time-limited test" daemon prio=5 tid=569 runnable
3274java.lang.Thread.State: RUNNABLE
3275 at java.lang.Thread.dumpThreads(Native Method)
3276 at java.lang.Thread.getAllStackTraces(Thread.java:1615)
3277 at org.apache.ozone.test.TimedOutTestsListener.buildThreadDump(TimedOutTestsListener.java:93)
3278 at org.apache.ozone.test.TimedOutTestsListener.buildThreadDiagnosticString(TimedOutTestsListener.java:79)
3279 at org.apache.ozone.test.GenericTestUtils.waitFor(GenericTestUtils.java:225)
3280 at org.apache.hadoop.fs.ozone.TestOzoneFileSystem.testTrash(TestOzoneFileSystem.java:1349)
3281 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
3282 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
3283 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
3284 at java.lang.reflect.Method.invoke(Method.java:498)
3285 at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
3286 at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
3287 at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
3288 at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
3289 at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
3290 at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:288)
3291 at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:282)
3292 at java.util.concurrent.FutureTask.run(FutureTask.java:266)
3293 at java.lang.Thread.run(Thread.java:750)
... {code}
was:
[https://github.com/kaijchen/ozone/runs/6150521370?check_suite_focus=true#step:4:2934]
[https://github.com/kaijchen/ozone/runs/6150521748?check_suite_focus=true#step:4:3350]
> Intermittent failure in TestOzoneFileSystem#testTrash
> -----------------------------------------------------
>
> Key: HDDS-6645
> URL: https://issues.apache.org/jira/browse/HDDS-6645
> Project: Apache Ozone
> Issue Type: Sub-task
> Reporter: Kaijie Chen
> Priority: Major
>
> [https://github.com/kaijchen/ozone/runs/6150521370?check_suite_focus=true#step:4:2934]
> {code:java}
> Error: Tests run: 136, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: 416.797 s <<< FAILURE! - in org.apache.hadoop.fs.ozone.TestOzoneFileSystem
> 2935Error: TestOzoneFileSystem.testTrash Time elapsed: 120.381 s <<< ERROR!
> 2936java.util.concurrent.TimeoutException:
> 2937Timed out waiting for condition. Thread diagnostics:
> 2938Timestamp: 2022-04-25 01:41:34,792
> 2939
> 2940"qtp1740509828-221-acceptor-0@6917bbb9-ServerConnector@6644d0cc{HTTP/1.1, (http/1.1)}{0.0.0.0:41183}" daemon prio=3 tid=221 runnable
> 2941java.lang.Thread.State: RUNNABLE
> 2942 at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
> 2943 at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421)
> 2944 at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249)
> 2945 at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
> 2946 at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
> 2947 at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
> 2948 at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
> 2949 at java.lang.Thread.run(Thread.java:750)
> 2950"IPC Server handler 8 on default port 41079" daemon prio=5 tid=61 timed_waiting
> 2951java.lang.Thread.State: TIMED_WAITING
> 2952 at sun.misc.Unsafe.park(Native Method)
> 2953 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 2954 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 2955 at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
> 2956 at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
> 2957 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2932)
> 2958"Datanode ReportManager Thread - 0" daemon prio=5 tid=205 in Object.wait()
> 2959java.lang.Thread.State: WAITING (on object monitor)
> 2960 at sun.misc.Unsafe.park(Native Method)
> 2961 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 2962 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 2963 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
> 2964 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> 2965 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 2966 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 2967 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 2968 at java.lang.Thread.run(Thread.java:750)
> 2969"IPC Server handler 14 on default port 41079" daemon prio=5 tid=67 timed_waiting
> 2970java.lang.Thread.State: TIMED_WAITING
> 2971 at sun.misc.Unsafe.park(Native Method)
> 2972 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 2973 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 2974 at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
> 2975 at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
> 2976 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2932)
> 2977"BlockDeletingService#2" daemon prio=5 tid=659 in Object.wait()
> 2978java.lang.Thread.State: WAITING (on object monitor)
> 2979 at sun.misc.Unsafe.park(Native Method)
> 2980 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 2981 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 2982 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
> 2983 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> 2984 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 2985 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 2986 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 2987 at java.lang.Thread.run(Thread.java:750)
> 2988"IPC Server handler 18 on default port 36721" daemon prio=5 tid=91 timed_waiting
> 2989java.lang.Thread.State: TIMED_WAITING
> 2990 at sun.misc.Unsafe.park(Native Method)
> 2991 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 2992 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 2993 at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
> 2994 at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
> 2995 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2932)
> 2996"BlockDeletingService#1" daemon prio=5 tid=308 in Object.wait()
> 2997java.lang.Thread.State: WAITING (on object monitor)
> 2998 at sun.misc.Unsafe.park(Native Method)
> 2999 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 3000 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 3001 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
> 3002 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> 3003 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 3004 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 3005 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 3006 at java.lang.Thread.run(Thread.java:750)
> 3007"qtp1849335409-249" daemon prio=5 tid=249 timed_waiting
> 3008java.lang.Thread.State: TIMED_WAITING
> 3009 at sun.misc.Unsafe.park(Native Method)
> 3010 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 3011 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 3012 at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
> 3013 at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
> 3014 at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
> 3015 at java.lang.Thread.run(Thread.java:750)
> 3016"BlockDeletingService#0" daemon prio=5 tid=327 timed_waiting
> 3017java.lang.Thread.State: TIMED_WAITING
> 3018 at sun.misc.Unsafe.park(Native Method)
> 3019 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 3020 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 3021 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
> 3022 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> 3023 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 3024 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 3025 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 3026 at java.lang.Thread.run(Thread.java:750)
> 3027"Datanode State Machine Task Thread - 1" prio=5 tid=319 in Object.wait()
> 3028java.lang.Thread.State: WAITING (on object monitor)
> 3029 at sun.misc.Unsafe.park(Native Method)
> 3030 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 3031 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 3032 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 3033 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 3034 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 3035 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 3036 at java.lang.Thread.run(Thread.java:750)
> 3037"BlockDeletingService#2" daemon prio=5 tid=656 in Object.wait()
> 3038java.lang.Thread.State: WAITING (on object monitor)
> 3039 at sun.misc.Unsafe.park(Native Method)
> 3040 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 3041 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 3042 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
> 3043 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> 3044 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 3045 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 3046 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 3047 at java.lang.Thread.run(Thread.java:750)
> 3048"pool-55-thread-3" prio=5 tid=587 in Object.wait()
> 3049java.lang.Thread.State: WAITING (on object monitor)
> 3050 at sun.misc.Unsafe.park(Native Method)
> 3051 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 3052 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 3053 at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
> 3054 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 3055 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 3056 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 3057 at java.lang.Thread.run(Thread.java:750)
> 3058"qtp1377489126-173" daemon prio=5 tid=173 timed_waiting
> 3059java.lang.Thread.State: TIMED_WAITING
> 3060 at sun.misc.Unsafe.park(Native Method)
> 3061 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 3062 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 3063 at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
> 3064 at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
> 3065 at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
> 3066 at java.lang.Thread.run(Thread.java:750)
> 3067"IPC Server handler 4 on default port 40135" daemon prio=5 tid=142 timed_waiting
> 3068java.lang.Thread.State: TIMED_WAITING
> 3069 at sun.misc.Unsafe.park(Native Method)
> 3070 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 3071 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 3072 at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
> 3073 at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
> 3074 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2932)
> 3075"IPC Server listener on 0" daemon prio=5 tid=117 runnable
> 3076java.lang.Thread.State: RUNNABLE
> 3077 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 3078 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 3079 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
> 3080 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 3081 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 3082 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
> 3083 at org.apache.hadoop.ipc.Server$Listener.run(Server.java:1350)
> 3084"IPC Server handler 17 on default port 35237" daemon prio=5 tid=50 timed_waiting
> 3085java.lang.Thread.State: TIMED_WAITING
> 3086 at sun.misc.Unsafe.park(Native Method)
> 3087 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 3088 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 3089 at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
> 3090 at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
> 3091 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2932)
> 3092"EndpointStateMachine task thread for /0.0.0.0:36721 - 0 " prio=5 tid=330 in Object.wait()
> 3093java.lang.Thread.State: WAITING (on object monitor)
> 3094 at sun.misc.Unsafe.park(Native Method)
> 3095 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 3096 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 3097 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 3098 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 3099 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 3100 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 3101 at java.lang.Thread.run(Thread.java:750)
> 3102"BlockDeletingService#0" daemon prio=5 tid=317 timed_waiting
> 3103java.lang.Thread.State: TIMED_WAITING
> 3104 at sun.misc.Unsafe.park(Native Method)
> 3105 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 3106 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 3107 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
> 3108 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> 3109 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 3110 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 3111 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 3112 at java.lang.Thread.run(Thread.java:750)
> 3113"qtp577672681-100" daemon prio=5 tid=100 timed_waiting
> 3114java.lang.Thread.State: TIMED_WAITING
> 3115 at sun.misc.Unsafe.park(Native Method)
> 3116 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 3117 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 3118 at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
> 3119 at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
> 3120 at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
> 3121 at java.lang.Thread.run(Thread.java:750)
> 3122"pool-152-thread-1" prio=5 tid=398 in Object.wait()
> 3123java.lang.Thread.State: WAITING (on object monitor)
> 3124 at sun.misc.Unsafe.park(Native Method)
> 3125 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 3126 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 3127 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 3128 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 3129 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 3130 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 3131 at java.lang.Thread.run(Thread.java:750)
> 3132"Datanode ReportManager Thread - 2" daemon prio=5 tid=180 in Object.wait()
> 3133java.lang.Thread.State: WAITING (on object monitor)
> 3134 at sun.misc.Unsafe.park(Native Method)
> 3135 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 3136 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 3137 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
> 3138 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> 3139 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 3140 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 3141 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 3142 at java.lang.Thread.run(Thread.java:750)
> 3143"d3eea043-2c2b-4d78-81f5-cd3edfbb9ed9@group-24FF75A3605B-StateMachineUpdater" daemon prio=5 tid=364 in Object.wait()
> 3144java.lang.Thread.State: WAITING (on object monitor)
> 3145 at java.lang.Object.wait(Native Method)
> 3146 at java.lang.Object.wait(Object.java:502)
> 3147 at org.apache.ratis.server.impl.StateMachineUpdater.waitForCommit(StateMachineUpdater.java:204)
> 3148 at org.apache.ratis.server.impl.StateMachineUpdater.run(StateMachineUpdater.java:173)
> 3149 at java.lang.Thread.run(Thread.java:750)
> 3150"Datanode State Machine Task Thread - 1" prio=5 tid=288 in Object.wait()
> 3151java.lang.Thread.State: WAITING (on object monitor)
> 3152 at sun.misc.Unsafe.park(Native Method)
> 3153 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 3154 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 3155 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 3156 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 3157 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 3158 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 3159 at java.lang.Thread.run(Thread.java:750)
> 3160"pool-145-thread-1" daemon prio=5 tid=266 timed_waiting
> 3161java.lang.Thread.State: TIMED_WAITING
> 3162 at sun.misc.Unsafe.park(Native Method)
> 3163 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 3164 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 3165 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
> 3166 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> 3167 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 3168 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 3169 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 3170 at java.lang.Thread.run(Thread.java:750)
> 3171"ContainerOp-b606fafc-cf14-4ded-8f6e-d56353c5b67d-6" prio=5 tid=523 in Object.wait()
> 3172java.lang.Thread.State: WAITING (on object monitor)
> 3173 at sun.misc.Unsafe.park(Native Method)
> 3174 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 3175 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 3176 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 3177 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 3178 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 3179 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 3180 at java.lang.Thread.run(Thread.java:750)
> 3181"surefire-forkedjvm-command-thread" daemon prio=5 tid=10 runnable
> 3182java.lang.Thread.State: RUNNABLE
> 3183 at java.io.FileInputStream.readBytes(Native Method)
> 3184 at java.io.FileInputStream.read(FileInputStream.java:255)
> 3185 at java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
> 3186 at java.io.BufferedInputStream.read(BufferedInputStream.java:345)
> 3187 at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
> 3188 at java.io.BufferedInputStream.read1(BufferedInputStream.java:286)
> 3189 at java.io.BufferedInputStream.read(BufferedInputStream.java:345)
> 3190 at org.apache.maven.surefire.api.util.internal.Channels$3.readImpl(Channels.java:214)
> 3191 at org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleReadableChannel.read(AbstractNoninterruptibleReadableChannel.java:54)
> 3192 at org.apache.maven.surefire.booter.spi.LegacyMasterProcessChannelDecoder.decode(LegacyMasterProcessChannelDecoder.java:80)
> 3193 at org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:343)
> 3194 at java.lang.Thread.run(Thread.java:750)
> 3195"0698bb1e-527a-4ce6-96df-4eb9049c94b6@group-24FF75A3605B->cadadf1f-4933-4fd8-9a68-0ae33772136e-GrpcLogAppender-LogAppenderDaemon" daemon prio=5 tid=579 timed_waiting
> 3196java.lang.Thread.State: TIMED_WAITING
> 3197 at java.lang.Object.wait(Native Method)
> 3198 at org.apache.ratis.grpc.server.GrpcLogAppender.mayWait(GrpcLogAppender.java:175)
> 3199 at org.apache.ratis.grpc.server.GrpcLogAppender.run(GrpcLogAppender.java:127)
> 3200 at org.apache.ratis.server.leader.LogAppenderDaemon.run(LogAppenderDaemon.java:77)
> 3201 at org.apache.ratis.server.leader.LogAppenderDaemon$$Lambda$859/1063038588.run(Unknown Source)
> 3202 at java.lang.Thread.run(Thread.java:750)
> 3203"d3eea043-2c2b-4d78-81f5-cd3edfbb9ed9@group-24FF75A3605B-FollowerState" daemon prio=5 tid=576 timed_waiting
> 3204java.lang.Thread.State: TIMED_WAITING
> 3205 at java.lang.Thread.sleep(Native Method)
> 3206 at java.lang.Thread.sleep(Thread.java:342)
> 3207 at java.util.concurrent.TimeUnit.sleep(TimeUnit.java:386)
> 3208 at org.apache.ratis.util.TimeDuration.sleep(TimeDuration.java:324)
> 3209 at org.apache.ratis.util.TimeDuration.sleep(TimeDuration.java:309)
> 3210 at org.apache.ratis.server.impl.FollowerState.run(FollowerState.java:118)
> 3211"ContainerOp-b606fafc-cf14-4ded-8f6e-d56353c5b67d-8" prio=5 tid=541 in Object.wait()
> 3212java.lang.Thread.State: WAITING (on object monitor)
> 3213 at sun.misc.Unsafe.park(Native Method)
> 3214 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 3215 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 3216 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 3217 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 3218 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 3219 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 3220 at java.lang.Thread.run(Thread.java:750)
> 3221"org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DeleteBlocksCommandHandler$DeleteCmdWorker@26dcac0d" daemon prio=5 tid=218 timed_waiting
> 3222java.lang.Thread.State: TIMED_WAITING
> 3223 at java.lang.Thread.sleep(Native Method)
> 3224 at org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DeleteBlocksCommandHandler$DeleteCmdWorker.run(DeleteBlocksCommandHandler.java:186)
> 3225 at java.lang.Thread.run(Thread.java:750)
> 3226"org.apache.hadoop.util.JvmPauseMonitor$Monitor@5b74c45a" daemon prio=5 tid=159 timed_waiting
> 3227java.lang.Thread.State: TIMED_WAITING
> 3228 at java.lang.Thread.sleep(Native Method)
> 3229 at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192)
> 3230 at java.lang.Thread.run(Thread.java:750)
> 3231"IPC Server handler 9 on default port 40135" daemon prio=5 tid=147 timed_waiting
> 3232java.lang.Thread.State: TIMED_WAITING
> 3233 at sun.misc.Unsafe.park(Native Method)
> 3234 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 3235 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 3236 at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
> 3237 at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
> 3238 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2932)
> 3239"IPC Server handler 3 on default port 41079" daemon prio=5 tid=56 timed_waiting
> 3240java.lang.Thread.State: TIMED_WAITING
> 3241 at sun.misc.Unsafe.park(Native Method)
> 3242 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 3243 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 3244 at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
> 3245 at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:317)
> 3246 at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2932)
> 3247"cadadf1f-4933-4fd8-9a68-0ae33772136e@group-C4594F0DA3B7-LeaderStateImpl" daemon prio=5 tid=408 timed_waiting
> 3248java.lang.Thread.State: TIMED_WAITING
> 3249 at sun.misc.Unsafe.park(Native Method)
> 3250 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 3251 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 3252 at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418)
> 3253 at org.apache.ratis.server.impl.LeaderStateImpl$EventQueue.poll(LeaderStateImpl.java:156)
> 3254 at org.apache.ratis.server.impl.LeaderStateImpl$EventProcessor.run(LeaderStateImpl.java:600)
> 3255"cadadf1f-4933-4fd8-9a68-0ae33772136e@group-24FF75A3605B-StateMachineUpdater" daemon prio=5 tid=354 in Object.wait()
> 3256java.lang.Thread.State: WAITING (on object monitor)
> 3257 at java.lang.Object.wait(Native Method)
> 3258 at java.lang.Object.wait(Object.java:502)
> 3259 at org.apache.ratis.server.impl.StateMachineUpdater.waitForCommit(StateMachineUpdater.java:204)
> 3260 at org.apache.ratis.server.impl.StateMachineUpdater.run(StateMachineUpdater.java:173)
> 3261 at java.lang.Thread.run(Thread.java:750)
> 3262"ChunkWriter-1-0" daemon prio=5 tid=322 in Object.wait()
> 3263java.lang.Thread.State: WAITING (on object monitor)
> 3264 at sun.misc.Unsafe.park(Native Method)
> 3265 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 3266 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 3267 at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
> 3268 at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
> 3269 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
> 3270 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
> 3271 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 3272 at java.lang.Thread.run(Thread.java:750)
> 3273"Time-limited test" daemon prio=5 tid=569 runnable
> 3274java.lang.Thread.State: RUNNABLE
> 3275 at java.lang.Thread.dumpThreads(Native Method)
> 3276 at java.lang.Thread.getAllStackTraces(Thread.java:1615)
> 3277 at org.apache.ozone.test.TimedOutTestsListener.buildThreadDump(TimedOutTestsListener.java:93)
> 3278 at org.apache.ozone.test.TimedOutTestsListener.buildThreadDiagnosticString(TimedOutTestsListener.java:79)
> 3279 at org.apache.ozone.test.GenericTestUtils.waitFor(GenericTestUtils.java:225)
> 3280 at org.apache.hadoop.fs.ozone.TestOzoneFileSystem.testTrash(TestOzoneFileSystem.java:1349)
> 3281 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 3282 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 3283 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 3284 at java.lang.reflect.Method.invoke(Method.java:498)
> 3285 at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
> 3286 at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> 3287 at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
> 3288 at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> 3289 at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
> 3290 at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:288)
> 3291 at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:282)
> 3292 at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> 3293 at java.lang.Thread.run(Thread.java:750)
> ... {code}
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org