You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@storm.apache.org by "Stig Rohde Døssing (JIRA)" <ji...@apache.org> on 2019/03/07 11:18:00 UTC
[jira] [Updated] (STORM-3354) LeaderElector is not shut down
properly
[ https://issues.apache.org/jira/browse/STORM-3354?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Stig Rohde Døssing updated STORM-3354:
--------------------------------------
Description:
Nimbus' LeaderElector is not shut down when Nimbus shuts down. This can cause test flakiness because the elector callback may be called after the Zookeeper client is closed.
Additionally the LeaderListenerCallback may in some cases quit the leadership election if Nimbus isn't ready to become master, but I don't see any code to re-enter the election.
{quote}java.lang.IllegalStateException: Client is not started
at org.apache.storm.shade.org.apache.curator.shaded.com.google.common.base.Preconditions.checkState(Preconditions.java:444) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:139) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:602) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1191) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1158) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.connection.StandardConnectionHandlingPolicy.callWithRetry(StandardConnectionHandlingPolicy.java:64) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:100) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.pathInForeground(CreateBuilderImpl.java:1155) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.protectedPathInForeground(CreateBuilderImpl.java:605) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.forPath(CreateBuilderImpl.java:595) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:360) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:308) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.zookeeper.ClientZookeeper.createNode(ClientZookeeper.java:98) ~[storm-client-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.nimbus.LeaderListenerCallback.setUpNimbusInfo(LeaderListenerCallback.java:154) ~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.nimbus.LeaderListenerCallback.leaderCallBack(LeaderListenerCallback.java:96) ~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.zookeeper.Zookeeper$1.isLeader(Zookeeper.java:123) ~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:665) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:661) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:93) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.shaded.com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:435) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:85) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.setLeadership(LeaderLatch.java:660) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.checkLeadership(LeaderLatch.java:539) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.access$700(LeaderLatch.java:65) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$7.processResult(LeaderLatch.java:590) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.sendToBackgroundCallback(CuratorFrameworkImpl.java:865) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.processBackgroundOperation(CuratorFrameworkImpl.java:635) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.WatcherRemovalFacade.processBackgroundOperation(WatcherRemovalFacade.java:152) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.GetChildrenBuilderImpl$2.processResult(GetChildrenBuilderImpl.java:187) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:590) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
{quote}
was:
Nimbus' LeaderElector is not shut down when Nimbus shuts down. This can cause test flakiness because the elector callback may be called after the Zookeeper client is closed.
{quote}
java.lang.IllegalStateException: Client is not started
at org.apache.storm.shade.org.apache.curator.shaded.com.google.common.base.Preconditions.checkState(Preconditions.java:444) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:139) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:602) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1191) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1158) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.connection.StandardConnectionHandlingPolicy.callWithRetry(StandardConnectionHandlingPolicy.java:64) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:100) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.pathInForeground(CreateBuilderImpl.java:1155) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.protectedPathInForeground(CreateBuilderImpl.java:605) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.forPath(CreateBuilderImpl.java:595) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:360) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:308) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.zookeeper.ClientZookeeper.createNode(ClientZookeeper.java:98) ~[storm-client-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.nimbus.LeaderListenerCallback.setUpNimbusInfo(LeaderListenerCallback.java:154) ~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.nimbus.LeaderListenerCallback.leaderCallBack(LeaderListenerCallback.java:96) ~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.zookeeper.Zookeeper$1.isLeader(Zookeeper.java:123) ~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:665) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:661) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:93) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.shaded.com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:435) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:85) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.setLeadership(LeaderLatch.java:660) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.checkLeadership(LeaderLatch.java:539) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.access$700(LeaderLatch.java:65) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$7.processResult(LeaderLatch.java:590) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.sendToBackgroundCallback(CuratorFrameworkImpl.java:865) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.processBackgroundOperation(CuratorFrameworkImpl.java:635) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.WatcherRemovalFacade.processBackgroundOperation(WatcherRemovalFacade.java:152) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.curator.framework.imps.GetChildrenBuilderImpl$2.processResult(GetChildrenBuilderImpl.java:187) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:590) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
{quote}
> LeaderElector is not shut down properly
> ---------------------------------------
>
> Key: STORM-3354
> URL: https://issues.apache.org/jira/browse/STORM-3354
> Project: Apache Storm
> Issue Type: Bug
> Affects Versions: 2.0.0
> Reporter: Stig Rohde Døssing
> Assignee: Stig Rohde Døssing
> Priority: Major
>
> Nimbus' LeaderElector is not shut down when Nimbus shuts down. This can cause test flakiness because the elector callback may be called after the Zookeeper client is closed.
> Additionally the LeaderListenerCallback may in some cases quit the leadership election if Nimbus isn't ready to become master, but I don't see any code to re-enter the election.
>
> {quote}java.lang.IllegalStateException: Client is not started
> at org.apache.storm.shade.org.apache.curator.shaded.com.google.common.base.Preconditions.checkState(Preconditions.java:444) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:139) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:602) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1191) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1158) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.connection.StandardConnectionHandlingPolicy.callWithRetry(StandardConnectionHandlingPolicy.java:64) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:100) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.pathInForeground(CreateBuilderImpl.java:1155) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.protectedPathInForeground(CreateBuilderImpl.java:605) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.forPath(CreateBuilderImpl.java:595) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:360) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:308) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.zookeeper.ClientZookeeper.createNode(ClientZookeeper.java:98) ~[storm-client-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.nimbus.LeaderListenerCallback.setUpNimbusInfo(LeaderListenerCallback.java:154) ~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.nimbus.LeaderListenerCallback.leaderCallBack(LeaderListenerCallback.java:96) ~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.zookeeper.Zookeeper$1.isLeader(Zookeeper.java:123) ~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:665) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:661) ~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:93) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.shaded.com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:435) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:85) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.setLeadership(LeaderLatch.java:660) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.checkLeadership(LeaderLatch.java:539) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.access$700(LeaderLatch.java:65) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$7.processResult(LeaderLatch.java:590) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.sendToBackgroundCallback(CuratorFrameworkImpl.java:865) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.processBackgroundOperation(CuratorFrameworkImpl.java:635) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.imps.WatcherRemovalFacade.processBackgroundOperation(WatcherRemovalFacade.java:152) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.curator.framework.imps.GetChildrenBuilderImpl$2.processResult(GetChildrenBuilderImpl.java:187) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:590) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498) [shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
> {quote}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)