You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@aurora.apache.org by "Bill Farner (JIRA)" <ji...@apache.org> on 2014/09/24 02:54:34 UTC
[jira] [Updated] (AURORA-746) LocalSchedulerMain shutdown deadlock
[ https://issues.apache.org/jira/browse/AURORA-746?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Bill Farner updated AURORA-746:
-------------------------------
Priority: Minor (was: Major)
> LocalSchedulerMain shutdown deadlock
> ------------------------------------
>
> Key: AURORA-746
> URL: https://issues.apache.org/jira/browse/AURORA-746
> Project: Aurora
> Issue Type: Story
> Components: Scheduler, Testing
> Reporter: Kevin Sweeney
> Priority: Minor
>
> Looks like LocalSchedulerMain waits forever on an attempt to cancel its zookeeper leadership
> {noformat}
> "qtp516227519-124" prio=10 tid=0x00007f65a8e16000 nid=0x4c50 waiting for monitor entry [0x00007f648faf5000]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at com.twitter.common.zookeeper.Group$ActiveMembership.cancel(Group.java:365)
> - waiting to lock <0x000000060f86d9a8> (a com.twitter.common.zookeeper.Group$ActiveMembership)
> at com.twitter.common.zookeeper.ServerSetImpl$MemberStatus.leave(ServerSetImpl.java:255)
> - locked <0x000000060f86ced0> (a com.twitter.common.zookeeper.ServerSetImpl$MemberStatus)
> at com.twitter.common.zookeeper.ServerSetImpl$2.leave(ServerSetImpl.java:199)
> at com.twitter.common.zookeeper.SingletonService$1$1.leave(SingletonService.java:185)
> - locked <0x000000060f86ce98> (a com.twitter.common.zookeeper.SingletonService$1$1)
> at org.apache.aurora.scheduler.SchedulerLifecycle$8.execute(SchedulerLifecycle.java:367)
> at org.apache.aurora.scheduler.SchedulerLifecycle$8.execute(SchedulerLifecycle.java:352)
> at com.twitter.common.base.Closures$4.execute(Closures.java:120)
> at com.twitter.common.base.Closures$3.execute(Closures.java:98)
> at com.twitter.common.util.StateMachine.transition(StateMachine.java:191)
> at org.apache.aurora.scheduler.SchedulerLifecycle$4.execute(SchedulerLifecycle.java:232)
> at com.twitter.common.application.ShutdownRegistry$ShutdownRegistryImpl.execute(ShutdownRegistry.java:88)
> - locked <0x000000060f86c6c0> (a com.twitter.common.application.ShutdownRegistry$ShutdownRegistryImpl)
> at com.twitter.common.application.Lifecycle.shutdown(Lifecycle.java:92)
> - locked <0x000000060f86c6b0> (a java.lang.Object)
> at org.apache.aurora.scheduler.storage.backup.BackupModule$LifecycleHook.execute(BackupModule.java:117)
> at org.apache.aurora.scheduler.storage.backup.Recovery$RecoveryImpl$PendingRecovery$1.execute(Recovery.java:198)
> at org.apache.aurora.scheduler.storage.Storage$MutateWork$NoResult.apply(Storage.java:132)
> at org.apache.aurora.scheduler.storage.Storage$MutateWork$NoResult$Quiet.apply(Storage.java:149)
> at org.apache.aurora.scheduler.storage.mem.MemStorage.doWork(MemStorage.java:175)
> at org.apache.aurora.scheduler.storage.mem.MemStorage.access$400(MemStorage.java:59)
> at org.apache.aurora.scheduler.storage.mem.MemStorage$4.apply(MemStorage.java:200)
> at org.apache.aurora.scheduler.storage.mem.MemStorage$4.apply(MemStorage.java:197)
> at org.apache.aurora.scheduler.storage.db.DbStorage.write(DbStorage.java:150)
> at org.apache.aurora.scheduler.storage.db.DbStorage$$EnhancerByGuice$$beec6366.CGLIB$write$4(<generated>)
> at org.apache.aurora.scheduler.storage.db.DbStorage$$EnhancerByGuice$$beec6366$$FastClassByGuice$$9e51bc15.invoke(<generated>)
> at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228)
> at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
> at org.mybatis.guice.transactional.TransactionalMethodInterceptor.invoke(TransactionalMethodInterceptor.java:101)
> at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
> at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:52)
> at org.apache.aurora.scheduler.storage.db.DbStorage$$EnhancerByGuice$$beec6366.write(<generated>)
> at org.apache.aurora.scheduler.storage.mem.MemStorage.write(MemStorage.java:197)
> at org.apache.aurora.scheduler.storage.backup.Recovery$RecoveryImpl$PendingRecovery.commit(Recovery.java:193)
> at org.apache.aurora.scheduler.storage.backup.Recovery$RecoveryImpl.commit(Recovery.java:182)
> at org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.commitRecovery(SchedulerThriftInterface.java:1009)
> at org.apache.aurora.scheduler.thrift.SchedulerThriftInterface$$EnhancerByGuice$$7f949b84.CGLIB$commitRecovery$7(<generated>)
> at org.apache.aurora.scheduler.thrift.SchedulerThriftInterface$$EnhancerByGuice$$7f949b84$$FastClassByGuice$$455b50c2.invoke(<generated>)
> at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228)
> at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
> at org.apache.aurora.scheduler.thrift.aop.ServerInfoInterceptor.invoke(ServerInfoInterceptor.java:32)
> at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
> at org.apache.aurora.scheduler.thrift.aop.ThriftStatsExporterInterceptor.invoke(ThriftStatsExporterInterceptor.java:47)
> at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
> at org.apache.aurora.scheduler.thrift.aop.FeatureToggleInterceptor.invoke(FeatureToggleInterceptor.java:38)
> at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
> at org.apache.aurora.scheduler.thrift.aop.UserCapabilityInterceptor.invoke(UserCapabilityInterceptor.java:92)
> at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
> at org.apache.aurora.scheduler.thrift.aop.LoggingInterceptor.invoke(LoggingInterceptor.java:87)
> at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:72)
> at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:52)
> at org.apache.aurora.scheduler.thrift.SchedulerThriftInterface$$EnhancerByGuice$$7f949b84.commitRecovery(<generated>)
> at org.apache.aurora.gen.AuroraAdmin$Processor$commitRecovery.getResult(AuroraAdmin.java:1296)
> at org.apache.aurora.gen.AuroraAdmin$Processor$commitRecovery.getResult(AuroraAdmin.java:1281)
> at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
> at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
> at org.apache.thrift.server.TServlet.doPost(TServlet.java:83)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:727)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
> at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:263)
> at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:178)
> at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:91)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:62)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:82)
> at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:294)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:168)
> at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58)
> at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:118)
> at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:113)
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1288)
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443)
> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044)
> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372)
> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
> at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)
> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
> at org.eclipse.jetty.server.Server.handle(Server.java:369)
> at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486)
> at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:944)
> at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1005)
> at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865)
> at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
> at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:667)
> at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
> {noformat}
> {noformat}
> "main-EventThread" daemon prio=10 tid=0x00007f65a8e52000 nid=0x4c56 waiting on condition [0x00007f648f4f3000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x000000076ed20ad8> (a java.util.concurrent.CountDownLatch$Sync)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
> at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834)
> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994)
> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303)
> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:236)
> at com.twitter.common.zookeeper.ZooKeeperClient.get(ZooKeeperClient.java:398)
> - locked <0x000000060f86d060> (a com.twitter.common.zookeeper.ZooKeeperClient)
> at com.twitter.common.zookeeper.ZooKeeperClient.get(ZooKeeperClient.java:332)
> - locked <0x000000060f86d060> (a com.twitter.common.zookeeper.ZooKeeperClient)
> at com.twitter.common.zookeeper.Group$ActiveMembership.join(Group.java:424)
> - locked <0x000000060f86d9a8> (a com.twitter.common.zookeeper.Group$ActiveMembership)
> at com.twitter.common.zookeeper.Group$ActiveMembership$4.get(Group.java:445)
> at com.twitter.common.zookeeper.Group$ActiveMembership$4.get(Group.java:442)
> at com.twitter.common.util.BackoffHelper$1.get(BackoffHelper.java:109)
> at com.twitter.common.util.BackoffHelper$1.get(BackoffHelper.java:107)
> at com.twitter.common.util.BackoffHelper.doUntilResult(BackoffHelper.java:127)
> at com.twitter.common.util.BackoffHelper.doUntilSuccess(BackoffHelper.java:107)
> at com.twitter.common.zookeeper.Group$ActiveMembership.tryJoin(Group.java:467)
> - locked <0x000000060f86d9a8> (a com.twitter.common.zookeeper.Group$ActiveMembership)
> at com.twitter.common.zookeeper.Group$ActiveMembership.access$800(Group.java:316)
> at com.twitter.common.zookeeper.Group$ActiveMembership$3.process(Group.java:433)
> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:507)
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)