You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/01/03 17:27:17 UTC
[JENKINS] Lucene-Solr-4.x-Linux (32bit/jdk1.6.0_45) - Build # 8843
- Failure!
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/8843/
Java: 32bit/jdk1.6.0_45 -server -XX:+UseConcMarkSweepGC
1 tests failed.
REGRESSION: org.apache.solr.cloud.OverseerTest.testOverseerFailure
Error Message:
KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
Stack Trace:
org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
at __randomizedtesting.SeedInfo.seed([694323355A1AF778:6D4BACC648BF1859]:0)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:428)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:425)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:382)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:369)
at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:112)
at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:165)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:109)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:157)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:157)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:277)
at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:153)
at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:584)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:662)
Build Log:
[...truncated 9894 lines...]
[junit4] Suite: org.apache.solr.cloud.OverseerTest
[junit4] 2> 825607 T3373 oas.SolrTestCaseJ4.initCore ####initCore
[junit4] 2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solrtest-OverseerTest-1388765673706
[junit4] 2> 825608 T3373 oas.SolrTestCaseJ4.initCore ####initCore end
[junit4] 2> 825613 T3373 oas.SolrTestCaseJ4.setUp ###Starting testShardAssignment
[junit4] 2> 825614 T3373 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 825614 T3374 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 825714 T3373 oasc.ZkTestServer.run start zk server on port:41045
[junit4] 2> 825716 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 825897 T3381 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1aa04c5 name:ZooKeeperConnection Watcher:127.0.0.1:41045 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 825897 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 825901 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 825902 T3384 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25eeb8 name:ZooKeeperConnection Watcher:127.0.0.1:41045 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 825903 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 825903 T3373 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 825908 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 825909 T3387 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f3ea90 name:ZooKeeperConnection Watcher:127.0.0.1:41045/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 825909 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 825910 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 825912 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 825913 T3390 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@8e7041 name:ZooKeeperConnection Watcher:127.0.0.1:41045/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 825914 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 825915 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 825919 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 825924 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 825928 T3373 oasc.Overseer.start Overseer (id=91014147192520707-127.0.0.1:41045_solr-n_0000000000) starting
[junit4] 2> 825930 T3373 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 825935 T3373 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 825939 T3373 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 825943 T3392 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 825944 T3373 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 825946 T3391 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 825949 T3373 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 825951 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 825956 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 825957 T3395 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@df8b06 name:ZooKeeperConnection Watcher:127.0.0.1:41045/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 825957 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 825959 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 825963 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1
[junit4] 2> 825968 T3387 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 825968 T3395 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 825971 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 825973 T3391 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 825974 T3391 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 825974 T3391 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3]
[junit4] 2> 825974 T3391 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 825977 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 825978 T3387 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 825978 T3395 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 826472 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 826524 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 826533 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 826537 T3391 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 826537 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 826542 T3391 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core2",
[junit4] 2> "core_node_name":"node2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 826542 T3391 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 826542 T3391 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
[junit4] 2> 826545 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 826649 T3395 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 826649 T3387 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 827037 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4] 2> 827047 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4] 2> 827054 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 827055 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 827055 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 827058 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 827059 T3391 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 827063 T3391 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core3",
[junit4] 2> "core_node_name":"node3",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 827064 T3391 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 827064 T3391 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
[junit4] 2> 827066 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 827170 T3387 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 827170 T3395 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 827559 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
[junit4] 2> 827574 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
[junit4] 2> 827588 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 827588 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 827589 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 827595 T3391 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 827596 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 827605 T3391 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core4",
[junit4] 2> "core_node_name":"node4",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 827605 T3391 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 827606 T3391 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 827610 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 827713 T3387 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 827713 T3395 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 828096 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 828107 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 828107 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 828107 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 828109 T3391 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 828110 T3391 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core5",
[junit4] 2> "core_node_name":"node5",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 828110 T3391 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 828110 T3391 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
[junit4] 2> 828113 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 828217 T3387 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 828217 T3395 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 828607 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4] 2> 828617 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 828618 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 828618 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 828620 T3391 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 828620 T3391 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core6",
[junit4] 2> "core_node_name":"node6",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 828621 T3391 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 828621 T3391 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
[junit4] 2> 828633 T3390 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 828740 T3387 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 828740 T3395 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 829118 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
[junit4] 2> 829129 T3395 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 829133 T3373 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:41045 41045
[junit4] 2> 829305 T3373 oas.SolrTestCaseJ4.tearDown ###Ending testShardAssignment
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 829310 T3373 oas.SolrTestCaseJ4.setUp ###Starting testDoubleAssignment
[junit4] 2> 829310 T3373 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 829311 T3396 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 829411 T3373 oasc.ZkTestServer.run start zk server on port:39102
[junit4] 2> 829412 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 829416 T3403 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18411ce name:ZooKeeperConnection Watcher:127.0.0.1:39102/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 829416 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 829417 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 829418 T3406 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@122aff4 name:ZooKeeperConnection Watcher:127.0.0.1:39102 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 829419 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 829458 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 829460 T3409 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@583b41 name:ZooKeeperConnection Watcher:127.0.0.1:39102 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 829460 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 829460 T3373 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 829464 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 829468 T3373 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 829471 T3373 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 829474 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 829479 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 829480 T3412 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14fc89d name:ZooKeeperConnection Watcher:127.0.0.1:39102/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 829480 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 829482 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 829486 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
[junit4] 2> 829505 T3403 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 829505 T3412 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 829507 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 829509 T3415 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78eb50 name:ZooKeeperConnection Watcher:127.0.0.1:39102/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 829509 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 829510 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 829514 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 829520 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 829523 T3373 oasc.Overseer.start Overseer (id=91014147435003908-127.0.0.1:39102_solr-n_0000000000) starting
[junit4] 2> 829527 T3373 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 829531 T3373 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 829538 T3373 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 829542 T3417 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 829545 T3416 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 829548 T3416 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 829549 T3416 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 829549 T3416 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
[junit4] 2> 829550 T3416 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 829553 T3415 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 829554 T3412 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 829554 T3403 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 830046 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 830066 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 830075 T3415 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 830076 T3373 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 830078 T3416 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 830082 T3412 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 830083 T3403 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 830086 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 830089 T3415 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 830089 T3420 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1738da9 name:ZooKeeperConnection Watcher:127.0.0.1:39102/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 830090 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 830093 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 830097 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
[junit4] 2> 830101 T3403 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 830101 T3420 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 830104 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 830105 T3415 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 830107 T3416 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 830107 T3416 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
[junit4] 2> 830112 T3415 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 830115 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 830122 T3415 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 830130 T3415 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 830234 T3420 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 830234 T3403 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 830238 T3373 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 830243 T3403 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 830243 T3420 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 830246 T3373 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:39102 39102
[junit4] 2> 830384 T3373 oas.SolrTestCaseJ4.tearDown ###Ending testDoubleAssignment
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 830401 T3373 oas.SolrTestCaseJ4.setUp ###Starting testOverseerFailure
[junit4] 2> 830401 T3373 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 830402 T3421 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 830502 T3373 oasc.ZkTestServer.run start zk server on port:38155
[junit4] 2> 830503 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 830506 T3428 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9a8653 name:ZooKeeperConnection Watcher:127.0.0.1:38155/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 830506 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 830507 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 830508 T3431 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1be23ae name:ZooKeeperConnection Watcher:127.0.0.1:38155 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 830509 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 830544 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 830545 T3434 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@fc8618 name:ZooKeeperConnection Watcher:127.0.0.1:38155 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 830545 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 830546 T3373 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 830550 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 830554 T3373 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 830560 T3373 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 830563 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 830567 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 830569 T3437 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f3db92 name:ZooKeeperConnection Watcher:127.0.0.1:38155/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 830570 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 830572 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 830575 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
[junit4] 2> 830580 T3428 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 830580 T3437 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 830581 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 830582 T3440 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18577b4 name:ZooKeeperConnection Watcher:127.0.0.1:38155/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 830582 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 830583 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 830586 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 830592 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 830597 T3373 oasc.Overseer.start Overseer (id=91014147506503684-127.0.0.1:38155_solr-n_0000000000) starting
[junit4] 2> 830599 T3373 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 830604 T3373 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 830610 T3373 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 830616 T3442 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 830618 T3441 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 830631 T3395 oasc.LeaderElector$1.process WARN org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard2/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:93)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
[junit4] 2>
[junit4] 2> 830631 T3395 oasc.LeaderElector$1.process WARN org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:93)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
[junit4] 2>
[junit4] 2> 830632 T3395 oasc.LeaderElector$1.process WARN org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard3/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:93)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
[junit4] 2>
[junit4] 2> 831619 T3440 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 831622 T3441 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 831623 T3441 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 831623 T3441 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
[junit4] 2> 831623 T3441 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 831629 T3440 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 831630 T3437 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 831630 T3428 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 832119 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 832132 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 832140 T3440 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 832141 T3373 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 832144 T3441 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 832147 T3440 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 832147 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 832151 T3441 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 832152 T3441 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
[junit4] 2> 832155 T3440 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 832260 T3437 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 832260 T3428 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 833316 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 833324 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 833326 T3445 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@180e58d name:ZooKeeperConnection Watcher:127.0.0.1:38155/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 833326 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 833331 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 833334 T3373 oasc.Overseer.start Overseer (id=91014147506503685-127.0.0.1:38155_solr-n_0000000001) starting
[junit4] 2> 833340 T3447 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 833343 T3446 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 833349 T3446 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 833350 T3446 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 833351 T3446 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
[junit4] 2> 833358 T3445 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 833360 T3437 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 833361 T3428 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 833417 T3445 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 833418 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 833421 T3446 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 833429 T3373 oasc.LeaderElector.checkIfIamLeader WARN Failed setting watch org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /collections/collection1/leader_elect/shard1/election/91014147506503683-node1_core1-n_0000000002
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1151)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:273)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:270)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:270)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:126)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:277)
[junit4] 2> at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:153)
[junit4] 2> at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:584)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:597)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
[junit4] 2> at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
[junit4] 2> at java.lang.Thread.run(Thread.java:662)
[junit4] 2>
[junit4] 2> 833433 T3437 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 833434 T3373 oasc.LeaderElector.checkIfIamLeader WARN Failed setting watch org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /collections/collection1/leader_elect/shard1/election/91014147506503683-node1_core1-n_0000000001
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1151)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:273)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:270)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:270)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:126)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:157)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:277)
[junit4] 2> at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:153)
[junit4] 2> at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:584)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:597)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
[junit4] 2> at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
[junit4] 2> at java.lang.Thread.run(Thread.java:662)
[junit4] 2>
[junit4] 2> 833434 T3446 oasc.SolrException.log ERROR Problem cleaning up collection in zk:collection1:org.apache.zookeeper.KeeperException$NotEmptyException: KeeperErrorCode = Directory not empty for /collections/collection1/leader_elect/shard1/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:125)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:173)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:170)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:170)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:654)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:649)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:649)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:649)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.removeCore(Overseer.java:875)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.processMessage(Overseer.java:226)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:187)
[junit4] 2> at java.lang.Thread.run(Thread.java:662)
[junit4] 2>
[junit4] 2> 833438 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 833440 T3445 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 833445 T3445 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 833448 T3446 oasc.Overseer$ClusterStateUpdater.setShardLeader ERROR Could not mark shard leader for non existing collection:collection1
[junit4] 2> 833449 T3437 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 833450 T3428 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 833454 T3373 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:38155 38155
[junit4] 2> 833548 T3373 oas.SolrTestCaseJ4.tearDown ###Ending testOverseerFailure
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=OverseerTest -Dtests.method=testOverseerFailure -Dtests.seed=694323355A1AF778 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar_LY -Dtests.timezone=America/Coral_Harbour -Dtests.file.encoding=UTF-8
[junit4] ERROR 3.18s J1 | OverseerTest.testOverseerFailure <<<
[junit4] > Throwable #1: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
[junit4] > at __randomizedtesting.SeedInfo.seed([694323355A1AF778:6D4BACC648BF1859]:0)
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] > at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:428)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:425)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:382)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:369)
[junit4] > at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:112)
[junit4] > at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:165)
[junit4] > at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:109)
[junit4] > at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:157)
[junit4] > at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:157)
[junit4] > at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:277)
[junit4] > at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:153)
[junit4] > at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:584)
[junit4] > at java.lang.Thread.run(Thread.java:662)
[junit4] 2> 833568 T3373 oas.SolrTestCaseJ4.setUp ###Starting testPlaceholders
[junit4] 2> 833569 T3373 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 833569 T3448 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 833669 T3373 oasc.ZkTestServer.run start zk server on port:38736
[junit4] 2> 833671 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 833673 T3455 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@bcfff1 name:ZooKeeperConnection Watcher:127.0.0.1:38736/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 833674 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 833675 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 833676 T3458 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@36633 name:ZooKeeperConnection Watcher:127.0.0.1:38736 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 833676 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 833708 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 833710 T3461 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1187e3b name:ZooKeeperConnection Watcher:127.0.0.1:38736 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 833710 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 833710 T3373 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 833727 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 833731 T3373 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 833734 T3373 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 833737 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 833741 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 833742 T3464 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@141a63a name:ZooKeeperConnection Watcher:127.0.0.1:38736/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 833743 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 833744 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 833748 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
[junit4] 2> 833752 T3455 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 833752 T3464 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 833753 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 833754 T3467 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@41045e name:ZooKeeperConnection Watcher:127.0.0.1:38736/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 833755 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 833756 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 833759 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 833764 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 833767 T3373 oasc.Overseer.start Overseer (id=91014147714121732-127.0.0.1:38736_solr-n_0000000000) starting
[junit4] 2> 833770 T3373 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 833774 T3373 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 833779 T3373 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 833783 T3469 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 833786 T3468 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 833790 T3468 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 833790 T3468 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=12 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"12",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 833791 T3468 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3, shard4, shard5, shard6, shard7, shard8, shard9, shard10, shard11, shard12]
[junit4] 2> 833791 T3468 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 833794 T3467 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 833795 T3464 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 833796 T3455 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 834288 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 834301 T3373 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 834310 T3467 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 834311 T3373 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 834314 T3468 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 834317 T3455 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 834317 T3464 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 834320 T3373 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:38736 38736
[junit4] 2> 834484 T3373 oas.SolrTestCaseJ4.tearDown ###Ending testPlaceholders
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 834491 T3373 oas.SolrTestCaseJ4.setUp ###Starting testShardAssignmentBigger
[junit4] 2> 834492 T3373 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 834492 T3470 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 834592 T3373 oasc.ZkTestServer.run start zk server on port:46668
[junit4] 2> 834593 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 834596 T3477 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4bf9a0 name:ZooKeeperConnection Watcher:127.0.0.1:46668 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 834597 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 834634 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 834636 T3480 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16b3461 name:ZooKeeperConnection Watcher:127.0.0.1:46668 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 834636 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 834636 T3373 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 834641 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 834642 T3483 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14a44e5 name:ZooKeeperConnection Watcher:127.0.0.1:46668/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 834643 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 834643 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 834646 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 834647 T3486 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e31105 name:ZooKeeperConnection Watcher:127.0.0.1:46668/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 834647 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 834649 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 834652 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 834657 T3373 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 834661 T3373 oasc.Overseer.start Overseer (id=91014147774545923-127.0.0.1:46668_solr-n_0000000000) starting
[junit4] 2> 834664 T3373 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 834675 T3373 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 834680 T3373 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 834684 T3488 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 834685 T3373 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 834686 T3487 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 834690 T3373 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 834692 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 834696 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 834698 T3491 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1d6375e name:ZooKeeperConnection Watcher:127.0.0.1:46668/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 834698 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 834700 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 834704 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/node0
[junit4] 2> 834716 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 834716 T3483 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 834717 T3491 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 834718 T3494 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2ab5ee name:ZooKeeperConnection Watcher:127.0.0.1:46668/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 834718 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 834720 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 834723 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
[junit4] 2> 834729 T3491 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 834730 T3483 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 834731 T3494 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 834732 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 834733 T3497 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e207d1 name:ZooKeeperConnection Watcher:127.0.0.1:46668/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 834733 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 834735 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 834739 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/node2
[junit4] 2> 834757 T3483 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 834757 T3494 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 834758 T3491 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 834759 T3497 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 834760 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 834761 T3500 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c6e068 name:ZooKeeperConnection Watcher:127.0.0.1:46668/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 834761 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 834763 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 834766 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/node3
[junit4] 2> 834771 T3483 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 834781 T3497 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 834781 T3500 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 834782 T3494 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 834782 T3491 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 834783 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 834784 T3503 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@197d9fb name:ZooKeeperConnection Watcher:127.0.0.1:46668/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 834784 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 834786 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 834789 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/node4
[junit4] 2> 834795 T3500 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 834795 T3483 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 834795 T3494 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 834796 T3503 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 834797 T3497 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 834797 T3491 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 834797 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 834798 T3506 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1237c2a name:ZooKeeperConnection Watcher:127.0.0.1:46668/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 834799 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 834800 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 834804 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/node5
[junit4] 2> 834810 T3503 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 834811 T3494 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 834812 T3500 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 834812 T3491 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 834812 T3506 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 834813 T3483 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 834814 T3497 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 834814 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 834815 T3509 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@891e72 name:ZooKeeperConnection Watcher:127.0.0.1:46668/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 834815 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 834817 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 834821 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/node6
[junit4] 2> 834827 T3494 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 834827 T3491 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 834828 T3483 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 834828 T3503 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 834828 T3506 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 834829 T3500 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 834830 T3497 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 834830 T3509 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 834831 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 834832 T3512 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@112c4fd name:ZooKeeperConnection Watcher:127.0.0.1:46668/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 834832 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 834835 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 834839 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/node7
[junit4] 2> 834845 T3494 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 834848 T3506 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 834848 T3483 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 834850 T3503 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 834854 T3500 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 834854 T3497 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 834855 T3491 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 834855 T3509 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 834857 T3512 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 834857 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 834858 T3515 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@be85ac name:ZooKeeperConnection Watcher:127.0.0.1:46668/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 834859 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 834860 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 834864 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/node8
[junit4] 2> 834872 T3483 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 834873 T3515 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 834874 T3506 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 834874 T3500 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 834874 T3494 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 834875 T3503 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 834875 T3491 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 834875 T3512 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 834876 T3509 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 834877 T3497 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 834878 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 834879 T3518 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14a8572 name:ZooKeeperConnection Watcher:127.0.0.1:46668/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 834880 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 834882 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 834886 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/node9
[junit4] 2> 834893 T3483 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 834896 T3503 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 834898 T3491 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 834903 T3497 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 834903 T3506 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 834904 T3512 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 834904 T3500 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 834906 T3518 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 834906 T3515 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 834906 T3494 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 834907 T3509 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 834907 T3373 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 834908 T3521 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@46c9ed name:ZooKeeperConnection Watcher:127.0.0.1:46668/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 834909 T3373 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 834911 T3373 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 834915 T3373 oascc.SolrZkClient.makePath makePath: /live_nodes/node10
[junit4] 2> 834926 T3491 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 834927 T3515 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 834927 T3500 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 834927 T3509 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 834927 T3503 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 834928 T3518 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 834928 T3506 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 834928 T3497 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 834928 T3494 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 834928 T3
[...truncated too long message...]
erElector.java:55)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
[junit4] 2>
[junit4] 2> 857668 T3373 oas.SolrTestCaseJ4.tearDown ###Ending testShardLeaderChange
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 857672 T3373 oas.SolrTestCaseJ4.initCore ####initCore
[junit4] 2> 857672 T3373 oas.SolrTestCaseJ4.initCore ####initCore end
[junit4] 2> 858114 T4039 oasc.LeaderElector$1.process WARN org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:93)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
[junit4] 2>
[junit4] 2> 858115 T4039 oasc.LeaderElector$1.process WARN org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:93)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
[junit4] 2>
[junit4] 2> 858646 T4050 oasc.LeaderElector$1.process WARN org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:93)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
[junit4] 2>
[junit4] 2> 858647 T4050 oasc.LeaderElector$1.process WARN org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:93)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
[junit4] 2>
[junit4] 2> 860672 T3373 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4] 2> NOTE: test params are: codec=Lucene42, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=ar_LY, timezone=America/Coral_Harbour
[junit4] 2> NOTE: Linux 3.8.0-34-generic i386/Sun Microsystems Inc. 1.6.0_45 (32-bit)/cpus=8,threads=1,free=124861344,total=202461184
[junit4] 2> NOTE: All tests run in this JVM: [PluginInfoTest, TestSolrCoreProperties, SolrRequestParserTest, TestSolrXmlPersistence, TestComponentsName, HdfsBasicDistributedZkTest, TestUtils, TestIndexingPerformance, DirectSolrConnectionTest, TestDistribDocBasedVersion, TestSimpleQParserPlugin, TestSchemaVersionResource, TestIBSimilarityFactory, TestLazyCores, TestRangeQuery, MinimalSchemaTest, TestReload, DateFieldTest, SuggesterWFSTTest, AliasIntegrationTest, CurrencyFieldOpenExchangeTest, TestUniqueKeyFieldResource, NumericFieldsTest, DefaultValueUpdateProcessorTest, AlternateDirectoryTest, SignatureUpdateProcessorFactoryTest, CoreMergeIndexesAdminHandlerTest, TestCollationField, TestDistributedSearch, TestHashPartitioner, TestDefaultSearchFieldResource, ZkSolrClientTest, TestZkChroot, SolrCmdDistributorTest, JSONWriterTest, DirectUpdateHandlerTest, TestFastLRUCache, TestPartialUpdateDeduplication, OpenCloseCoreStressTest, TestFiltering, TestIndexSearcher, TestDefaultSimilarityFactory, ChaosMonkeyNothingIsSafeTest, ConnectionManagerTest, TestNoOpRegenerator, SimplePostToolTest, BlockCacheTest, ShardRoutingCustomTest, SolrPluginUtilsTest, RequiredFieldsTest, FileUtilsTest, TestCollapseQParserPlugin, TestSolrDeletionPolicy2, TestCSVLoader, TestRTGBase, TestDistributedMissingSort, SpellCheckComponentTest, PreAnalyzedFieldTest, TestQuerySenderNoQuery, TestDynamicFieldCollectionResource, SpellingQueryConverterTest, TestCloudManagedSchemaAddField, TestSolr4Spatial, PathHierarchyTokenizerFactoryTest, TestSolrQueryParserDefaultOperatorResource, PrimUtilsTest, OpenExchangeRatesOrgProviderTest, ConvertedLegacyTest, TestPhraseSuggestions, DocValuesMultiTest, CoreContainerCoreInitFailuresTest, DocumentAnalysisRequestHandlerTest, TestFastWriter, PingRequestHandlerTest, TestBinaryField, OutputWriterTest, SolrIndexConfigTest, TestFieldCollectionResource, TermVectorComponentDistributedTest, TestImplicitCoreProperties, UniqFieldsUpdateProcessorFactoryTest, SpellCheckCollatorTest, TestXIncludeConfig, HdfsChaosMonkeySafeLeaderTest, BasicDistributedZkTest, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, OverseerTest]
[junit4] Completed on J1 in 35.11s, 8 tests, 1 error <<< FAILURES!
[...truncated 518 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:459: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:439: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:491: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1307: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:940: There were test failures: 348 suites, 1522 tests, 1 error, 36 ignored (7 assumptions)
Total time: 51 minutes 14 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.6.0_45 -server -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure