You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2015/03/02 00:24:30 UTC
[JENKINS] Lucene-Solr-4.10-Linux (64bit/jdk1.8.0_31) - Build #
11741 - Failure!
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.10-Linux/46/
Java: 64bit/jdk1.8.0_31 -XX:+UseCompressedOops -XX:+UseSerialGC
1 tests failed.
FAILED: org.apache.solr.cloud.OverseerTest.testOverseerFailure
Error Message:
KeeperErrorCode = NoNode for /collections/collection1/leader_elect/shard1/election
Stack Trace:
org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /collections/collection1/leader_elect/shard1/election
at __randomizedtesting.SeedInfo.seed([AD0DB48651D1258:ED854BB77B8FD79]:0)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:291)
at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:288)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:288)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:155)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:314)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:221)
at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:157)
at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:662)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 11476 lines...]
[junit4] Suite: org.apache.solr.cloud.OverseerTest
[junit4] 2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/solr/build/solr-core/test/J0/./solr.cloud.OverseerTest-AD0DB48651D1258-001/init-core-data-001
[junit4] 2> 1751188 T4584 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
[junit4] 2> 1751188 T4584 oas.SolrTestCaseJ4.initCore ####initCore
[junit4] 2> 1751188 T4584 oas.SolrTestCaseJ4.initCore ####initCore end
[junit4] 2> 1751190 T4584 oas.SolrTestCaseJ4.setUp ###Starting testShardAssignment
[junit4] 2> 1751190 T4584 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1751191 T4585 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 1751291 T4584 oasc.ZkTestServer.run start zk server on port:60483
[junit4] 2> 1751292 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1751294 T4591 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@11b03e54 name:ZooKeeperConnection Watcher:127.0.0.1:60483 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1751294 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1751295 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1751296 T4593 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c2894e6 name:ZooKeeperConnection Watcher:127.0.0.1:60483 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1751296 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1751296 T4584 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 1751298 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1751299 T4595 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@207fb947 name:ZooKeeperConnection Watcher:127.0.0.1:60483/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1751299 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1751299 T4584 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 1751301 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1751301 T4597 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@139f625 name:ZooKeeperConnection Watcher:127.0.0.1:60483/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1751302 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1751302 T4584 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: retry=true
[junit4] 2> 1751303 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 1751305 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 1751306 T4584 oasc.Overseer.close Overseer (id=null) closing
[junit4] 2> 1751307 T4584 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:60483_solr
[junit4] 2> 1751307 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 1751308 T4584 oasc.Overseer.start Overseer (id=93405231365029891-127.0.0.1:60483_solr-n_0000000000) starting
[junit4] 2> 1751309 T4584 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 1751310 T4584 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 1751312 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
[junit4] 2> 1751313 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
[junit4] 2> 1751314 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
[junit4] 2> 1751315 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 1751317 T4584 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 1751317 T4599 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 1751318 T4584 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 1751318 T4598 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 1751318 T4584 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 1751319 T4584 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1751320 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1751321 T4602 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5414365e name:ZooKeeperConnection Watcher:127.0.0.1:60483/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1751321 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1751322 T4584 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1751323 T4584 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1
[junit4] 2> 1751325 T4605 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1751326 T4598 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 1751326 T4598 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3]
[junit4] 2> 1751326 T4598 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
[junit4] 2> 1751327 T4603 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1751327 T4604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1751825 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4] 2> 1751830 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4] 2> 1751832 T4605 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1751834 T4598 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core2",
[junit4] 2> "core_node_name":"node2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 1751834 T4598 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 1751834 T4598 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
[junit4] 2> 1751835 T4605 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1751835 T4603 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1751836 T4604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1752333 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
[junit4] 2> 1752336 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
[junit4] 2> 1752338 T4605 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1752340 T4598 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core3",
[junit4] 2> "core_node_name":"node3",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 1752341 T4598 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 1752341 T4598 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 1752341 T4605 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1752342 T4603 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1752342 T4604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1752839 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 1752843 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1752845 T4605 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1752847 T4598 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core4",
[junit4] 2> "core_node_name":"node4",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 1752847 T4598 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 1752847 T4598 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
[junit4] 2> 1752848 T4605 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1752848 T4603 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1752848 T4604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1753346 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4] 2> 1753350 T4605 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1753351 T4598 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core5",
[junit4] 2> "core_node_name":"node5",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 1753351 T4598 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 1753351 T4598 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
[junit4] 2> 1753453 T4603 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1753453 T4604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1753850 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
[junit4] 2> 1753855 T4605 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1753864 T4598 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core6",
[junit4] 2> "core_node_name":"node6",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 1753865 T4598 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 1753865 T4598 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 1753968 T4604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1753968 T4603 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1754355 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 1754359 T4602 oaz.ClientCnxn$EventThread.processEvent ERROR Error while calling watcher java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@74bbfadb rejected from java.util.concurrent.ThreadPoolExecutor@4786a36d[Shutting down, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 8]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2047)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
[junit4] 2> at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:207)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
[junit4] 2>
[junit4] 2> 1754359 T4606 oasc.LeaderElector$ElectionWatcher.process WARN org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard2/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:291)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:288)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:288)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3$1.run(SolrZkClient.java:210)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 1755859 T4603 oasc.LeaderElector$ElectionWatcher.process WARN org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:291)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:288)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:288)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3$1.run(SolrZkClient.java:210)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 1755862 T4584 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:60483 60483
[junit4] 2> 1756430 T4585 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:60483 60483
[junit4] 2> 1756431 T4584 oas.SolrTestCaseJ4.tearDown ###Ending testShardAssignment
[junit4] 2> 1756432 T4584 oasc.Overseer.close Overseer (id=93405231365029891-127.0.0.1:60483_solr-n_0000000000) closing
[junit4] 2> 1756432 T4598 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:60483_solr
[junit4] 2> 1756433 T4608 oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read the data org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:308)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:321)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:90)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:293)
[junit4] 2>
[junit4] 2> 1756436 T4584 oas.SolrTestCaseJ4.setUp ###Starting testPlaceholders
[junit4] 2> 1756436 T4584 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1756437 T4609 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 1756536 T4584 oasc.ZkTestServer.run start zk server on port:41025
[junit4] 2> 1756537 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1756539 T4615 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5227aecf name:ZooKeeperConnection Watcher:127.0.0.1:41025/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1756539 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1756540 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1756541 T4617 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3bafee97 name:ZooKeeperConnection Watcher:127.0.0.1:41025 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1756541 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1756542 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1756543 T4619 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4efc897a name:ZooKeeperConnection Watcher:127.0.0.1:41025 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1756543 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1756544 T4584 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 1756546 T4584 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 1756547 T4584 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 1756548 T4584 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 1756548 T4584 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1756550 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1756550 T4621 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@47f6c1d0 name:ZooKeeperConnection Watcher:127.0.0.1:41025/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1756551 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1756551 T4584 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1756552 T4584 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
[junit4] 2> 1756556 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1756557 T4625 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@28c85e37 name:ZooKeeperConnection Watcher:127.0.0.1:41025/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1756557 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1756557 T4584 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: retry=true
[junit4] 2> 1756558 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 1756559 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 1756560 T4584 oasc.Overseer.close Overseer (id=null) closing
[junit4] 2> 1756561 T4584 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:41025_solr
[junit4] 2> 1756561 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 1756562 T4584 oasc.Overseer.start Overseer (id=93405231708831748-127.0.0.1:41025_solr-n_0000000000) starting
[junit4] 2> 1756562 T4584 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 1756564 T4584 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 1756565 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
[junit4] 2> 1756566 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
[junit4] 2> 1756567 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
[junit4] 2> 1756568 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 1756570 T4584 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 1756570 T4627 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 1756571 T4626 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 1756572 T4629 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1756572 T4626 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=12 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"12",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 1756573 T4626 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3, shard4, shard5, shard6, shard7, shard8, shard9, shard10, shard11, shard12]
[junit4] 2> 1756573 T4626 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
[junit4] 2> 1756574 T4622 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1756574 T4623 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1757072 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4] 2> 1757076 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4] 2> 1757078 T4629 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1757080 T4584 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:41025 41025
[junit4] 2> 1757952 T4609 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:41025 41025
[junit4] 2> 1757953 T4584 oas.SolrTestCaseJ4.tearDown ###Ending testPlaceholders
[junit4] 2> 1757953 T4584 oasc.Overseer.close Overseer (id=93405231708831748-127.0.0.1:41025_solr-n_0000000000) closing
[junit4] 2> 1757954 T4626 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:41025_solr
[junit4] 2> 1757955 T4630 oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read the data org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:308)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:321)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:90)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:293)
[junit4] 2>
[junit4] 2> 1757957 T4584 oas.SolrTestCaseJ4.setUp ###Starting testDoubleAssignment
[junit4] 2> 1757958 T4584 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1757958 T4631 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 1758058 T4584 oasc.ZkTestServer.run start zk server on port:33044
[junit4] 2> 1758059 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1758060 T4637 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3743940e name:ZooKeeperConnection Watcher:127.0.0.1:33044/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1758061 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1758061 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1758062 T4639 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5d905d37 name:ZooKeeperConnection Watcher:127.0.0.1:33044 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1758062 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1758063 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1758064 T4641 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@fa288d4 name:ZooKeeperConnection Watcher:127.0.0.1:33044 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1758064 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1758064 T4584 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 1758066 T4584 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 1758067 T4584 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 1758068 T4584 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 1758068 T4584 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1758069 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1758070 T4643 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1855ec63 name:ZooKeeperConnection Watcher:127.0.0.1:33044/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1758070 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1758070 T4584 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1758071 T4584 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
[junit4] 2> 1758073 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1758073 T4647 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@331df883 name:ZooKeeperConnection Watcher:127.0.0.1:33044/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1758073 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1758074 T4584 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: retry=true
[junit4] 2> 1758075 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 1758076 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 1758077 T4584 oasc.Overseer.close Overseer (id=null) closing
[junit4] 2> 1758077 T4584 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:33044_solr
[junit4] 2> 1758078 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 1758078 T4584 oasc.Overseer.start Overseer (id=93405231808512004-127.0.0.1:33044_solr-n_0000000000) starting
[junit4] 2> 1758079 T4584 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 1758080 T4584 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 1758081 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
[junit4] 2> 1758082 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
[junit4] 2> 1758083 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
[junit4] 2> 1758084 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 1758086 T4584 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 1758087 T4649 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 1758087 T4648 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 1758088 T4648 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 1758088 T4648 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
[junit4] 2> 1758088 T4648 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 1758089 T4651 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1758090 T4645 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1758089 T4644 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1758587 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 1758591 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1758593 T4651 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1758596 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1758596 T4653 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40456f26 name:ZooKeeperConnection Watcher:127.0.0.1:33044/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1758596 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1758599 T4584 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1758600 T4584 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
[junit4] 2> 1758604 T4651 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1758604 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 1758604 T4648 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 1758604 T4648 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
[junit4] 2> 1758606 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1758608 T4651 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1758710 T4644 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1758710 T4654 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1758712 T4632 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x14bd7903e1c0004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 1758714 T4584 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:33044 33044
[junit4] 2> 1759325 T4631 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:33044 33044
[junit4] 2> 1759326 T4584 oas.SolrTestCaseJ4.tearDown ###Ending testDoubleAssignment
[junit4] 2> 1759326 T4584 oasc.Overseer.close Overseer (id=93405231808512004-127.0.0.1:33044_solr-n_0000000000) closing
[junit4] 2> 1759327 T4648 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:33044_solr
[junit4] 2> 1759328 T4655 oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read the data org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:308)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:321)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:90)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:293)
[junit4] 2>
[junit4] 2> 1759331 T4584 oas.SolrTestCaseJ4.setUp ###Starting testBadQueueItem
[junit4] 2> 1759332 T4584 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1759332 T4656 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 1759432 T4584 oasc.ZkTestServer.run start zk server on port:46375
[junit4] 2> 1759433 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1759434 T4662 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2bdd1505 name:ZooKeeperConnection Watcher:127.0.0.1:46375 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1759435 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1759436 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1759437 T4664 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@79d1b55c name:ZooKeeperConnection Watcher:127.0.0.1:46375 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1759437 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1759437 T4584 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 1759439 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1759440 T4666 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1206c91f name:ZooKeeperConnection Watcher:127.0.0.1:46375/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1759440 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1759440 T4584 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 1759441 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1759442 T4668 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c2adf34 name:ZooKeeperConnection Watcher:127.0.0.1:46375/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1759442 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1759442 T4584 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: retry=true
[junit4] 2> 1759443 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 1759444 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 1759445 T4584 oasc.Overseer.close Overseer (id=null) closing
[junit4] 2> 1759446 T4584 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:46375_solr
[junit4] 2> 1759447 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 1759447 T4584 oasc.Overseer.start Overseer (id=93405231898558467-127.0.0.1:46375_solr-n_0000000000) starting
[junit4] 2> 1759448 T4584 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 1759449 T4584 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 1759451 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
[junit4] 2> 1759452 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
[junit4] 2> 1759453 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
[junit4] 2> 1759454 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 1759456 T4584 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 1759457 T4670 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 1759457 T4584 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 1759458 T4669 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 1759458 T4584 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 1759459 T4584 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1759460 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1759461 T4673 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ff9c92f name:ZooKeeperConnection Watcher:127.0.0.1:46375/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1759461 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1759462 T4584 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1759463 T4584 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1
[junit4] 2> 1759465 T4676 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1759466 T4669 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 1759466 T4669 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3]
[junit4] 2> 1759466 T4669 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
[junit4] 2> 1759467 T4675 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1759467 T4674 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1759965 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4] 2> 1759969 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4] 2> 1759971 T4676 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1759973 T4669 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core2",
[junit4] 2> "core_node_name":"node2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 1759973 T4669 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 1759973 T4669 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
[junit4] 2> 1759974 T4676 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1759974 T4675 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1759974 T4674 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1760472 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
[junit4] 2> 1760475 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
[junit4] 2> 1760477 T4676 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1760479 T4669 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core3",
[junit4] 2> "core_node_name":"node3",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 1760479 T4669 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 1760479 T4669 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 1760480 T4676 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1760480 T4675 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1760480 T4674 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1760978 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 1760981 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1760983 T4676 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1761085 T4675 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1761085 T4674 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1761134 T4676 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1761136 T4669 oasc.Overseer$ClusterStateUpdater.checkKeyExistence ERROR Skipping invalid Overseer message because it has no collection specified: {
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core0",
[junit4] 2> "core_node_name":"node0",
[junit4] 2> "collection":"",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 1761140 T4669 oasc.Overseer$ClusterStateUpdater.checkKeyExistence ERROR Skipping invalid Overseer message because it has no collection specified: {
[junit4] 2> "operation":"deletecore",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core0",
[junit4] 2> "core_node_name":"node0",
[junit4] 2> "collection":""}
[junit4] 2> 1761141 T4676 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1761142 T4669 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection2",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 1761142 T4669 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection2 with shards [shard1, shard2, shard3]
[junit4] 2> 1761142 T4669 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
[junit4] 2> 1761244 T4674 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1761244 T4675 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1761636 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection2/leader_elect/shard2/election
[junit4] 2> 1761640 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection2/leaders/shard2
[junit4] 2> 1761642 T4676 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1761644 T4669 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core2",
[junit4] 2> "core_node_name":"node2",
[junit4] 2> "collection":"collection2",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 1761645 T4669 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 1761645 T4669 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
[junit4] 2> 1761646 T4676 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1761646 T4674 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1761646 T4675 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1762143 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection2/leader_elect/shard3/election
[junit4] 2> 1762146 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection2/leaders/shard3
[junit4] 2> 1762148 T4676 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1762150 T4669 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core3",
[junit4] 2> "core_node_name":"node3",
[junit4] 2> "collection":"collection2",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 1762151 T4669 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 1762151 T4669 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 1762153 T4676 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1762154 T4675 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1762154 T4674 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1762649 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection2/leader_elect/shard1/election
[junit4] 2> 1762653 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection2/leaders/shard1
[junit4] 2> 1762654 T4676 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1762757 T4674 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1762757 T4675 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1762807 T4584 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:46375 46375
[junit4] 2> 1763459 T4670 oasc.OverseerCollectionProcessor.amILeader org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:308)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.cloud.OverseerCollectionProcessor.amILeader(OverseerCollectionProcessor.java:515)
[junit4] 2> at org.apache.solr.cloud.OverseerCollectionProcessor.run(OverseerCollectionProcessor.java:273)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 1763743 T4670 oasc.OverseerCollectionProcessor.amILeader According to ZK I (id=93405231898558467-127.0.0.1:46375_solr-n_0000000000) am no longer a leader.
[junit4] 2> 1763744 T4656 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:46375 46375
[junit4] 2> 1763745 T4584 oas.SolrTestCaseJ4.tearDown ###Ending testBadQueueItem
[junit4] 2> 1763745 T4584 oasc.Overseer.close Overseer (id=93405231898558467-127.0.0.1:46375_solr-n_0000000000) closing
[junit4] 2> 1763746 T4669 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:46375_solr
[junit4] 2> 1763749 T4677 oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read the data org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:308)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:321)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:90)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:293)
[junit4] 2>
[junit4] 2> 1763750 T4584 oas.SolrTestCaseJ4.setUp ###Starting testOverseerFailure
[junit4] 2> 1763751 T4584 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1763751 T4678 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 1763851 T4584 oasc.ZkTestServer.run start zk server on port:36296
[junit4] 2> 1763852 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1763854 T4684 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e739862 name:ZooKeeperConnection Watcher:127.0.0.1:36296 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1763854 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1763856 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1763856 T4686 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@48cd24e6 name:ZooKeeperConnection Watcher:127.0.0.1:36296 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1763857 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1763857 T4584 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 1763859 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1763860 T4688 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@192c3c61 name:ZooKeeperConnection Watcher:127.0.0.1:36296/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1763860 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1763860 T4584 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 1763861 T4584 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 1763862 T4584 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 1763863 T4584 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1763864 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1763865 T4690 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fae4571 name:ZooKeeperConnection Watcher:127.0.0.1:36296/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1763865 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1763865 T4584 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1763866 T4584 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
[junit4] 2> 1763871 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1763881 T4694 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2d6c666e name:ZooKeeperConnection Watcher:127.0.0.1:36296/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1763881 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1763882 T4584 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: retry=true
[junit4] 2> 1763883 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 1763886 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 1763888 T4584 oasc.Overseer.close Overseer (id=null) closing
[junit4] 2> 1763889 T4584 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:36296_solr
[junit4] 2> 1763889 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 1763891 T4584 oasc.Overseer.start Overseer (id=93405232188162052-127.0.0.1:36296_solr-n_0000000000) starting
[junit4] 2> 1763892 T4584 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 1763895 T4584 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 1763897 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
[junit4] 2> 1763901 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
[junit4] 2> 1763903 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
[junit4] 2> 1763906 T4584 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 1763908 T4584 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 1763909 T4696 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 1763909 T4695 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 1764910 T4698 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1764912 T4695 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 1764912 T4695 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
[junit4] 2> 1764912 T4695 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 1764913 T4691 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1764914 T4692 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1765410 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 1765417 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1765421 T4698 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1765423 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 1765423 T4695 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 1765424 T4695 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
[junit4] 2> 1765425 T4698 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1765426 T4691 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1765426 T4692 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1765910 T4696 oasc.OverseerCollectionProcessor.amILeader org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:308)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.cloud.OverseerCollectionProcessor.amILeader(OverseerCollectionProcessor.java:515)
[junit4] 2> at org.apache.solr.cloud.OverseerCollectionProcessor.run(OverseerCollectionProcessor.java:273)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 1765911 T4696 oasc.OverseerCollectionProcessor.amILeader According to ZK I (id=93405232188162052-127.0.0.1:36296_solr-n_0000000000) am no longer a leader.
[junit4] 2> 1766429 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 1766432 T4584 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1766433 T4700 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@593821de name:ZooKeeperConnection Watcher:127.0.0.1:36296/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1766433 T4584 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1766434 T4584 oasc.Overseer.close Overseer (id=93405232188162052-127.0.0.1:36296_solr-n_0000000000) closing
[junit4] 2> 1766434 T4695 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:36296_solr
[junit4] 2> 1766434 T4584 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: retry=true
[junit4] 2> 1766436 T4701 oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read the data org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:308)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:321)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:90)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:293)
[junit4] 2>
[junit4] 2> 1766439 T4584 oasc.Overseer.close Overseer (id=null) closing
[junit4] 2> 1766441 T4584 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:36296_solr
[junit4] 2> 1766441 T4584 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 1766443 T4584 oasc.Overseer.start Overseer (id=93405232188162053-127.0.0.1:36296_solr-n_0000000001) starting
[junit4] 2> 1766448 T4584 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 1766449 T4703 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 1766450 T4702 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 1766453 T4702 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 1766453 T4702 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
[junit4] 2> 1766454 T4705 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1766455 T4691 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1766455 T4692 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1766506 T4705 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
[junit4] 2> 1766506 T4584 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 1766509 T4584 oasc.LeaderElector.checkIfIamLeader WARN Failed setting watch org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /collections/collection1/leader_elect/shard1/election/93405232188162051-node1_core1-n_0000000002
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:308)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:148)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:314)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:221)
[junit4] 2> at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:157)
[junit4] 2> at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:662)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:483)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
[junit4] 2> at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 1766510 T4706 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1766510 T4692 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1766513 T4707 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4] 2> 1766614 T4691 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
[junit4] 2> 1768013 T4692 oasc.LeaderElector$ElectionWatcher.process WARN org.apache.solr.common.SolrException: Could not register as the leader because creating the ephemeral registration node in ZooKeeper failed
[junit4] 2> at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:146)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:163)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:125)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3$1.run(SolrZkClient.java:210)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1
[junit4] 2> at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:40)
[junit4] 2> at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:134)
[junit4] 2> ... 10 more
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$11.execute(SolrZkClient.java:463)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkCl
[...truncated too long message...]
tionProcessor.run Process current queue of collection creations
[junit4] 2> 1808367 T5273 oasc.Overseer$ClusterStateUpdater.run Replaying operations from work queue.
[junit4] 2> 1808367 T5273 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"state1",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 1808367 T5273 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
[junit4] 2> 1808367 T5268 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1808368 T5273 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"state2",
[junit4] 2> "node_name":"node2",
[junit4] 2> "core":"core4",
[junit4] 2> "core_node_name":"node2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node2/solr/"}
[junit4] 2> 1808368 T5273 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
[junit4] 2> 1808369 T5268 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1808370 T5268 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1808371 T5273 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 1809848 T5265 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4] 2> 1810367 T5274 oasc.OverseerCollectionProcessor.amILeader org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:308)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.cloud.OverseerCollectionProcessor.amILeader(OverseerCollectionProcessor.java:515)
[junit4] 2> at org.apache.solr.cloud.OverseerCollectionProcessor.run(OverseerCollectionProcessor.java:273)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 1810367 T5274 oasc.OverseerCollectionProcessor.amILeader According to ZK I (id=93405233301028912-127.0.0.1:41617_solr-n_0000000015) am no longer a leader.
[junit4] 2> 1811369 T5268 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4] 2> 1811370 T4584 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:41617 41617
[junit4] 2> 1812093 T5048 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:41617 41617
[junit4] 2> 1812094 T4584 oas.SolrTestCaseJ4.tearDown ###Ending testShardLeaderChange
[junit4] 2> 1812096 T4584 oasc.Overseer.close Overseer (id=93405233301028912-127.0.0.1:41617_solr-n_0000000015) closing
[junit4] 2> 1812097 T5273 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:41617_solr
[junit4] 2> 1812097 T5276 oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read the data org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:308)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:305)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:321)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:90)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:293)
[junit4] 2>
[junit4] 2> 1815098 T4584 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/solr/build/solr-core/test/J0/./solr.cloud.OverseerTest-AD0DB48651D1258-001
[junit4] 2> NOTE: test params are: codec=Lucene40, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=ro_RO, timezone=Asia/Srednekolymsk
[junit4] 2> NOTE: Linux 3.13.0-46-generic amd64/Oracle Corporation 1.8.0_31 (64-bit)/cpus=8,threads=1,free=53283008,total=258818048
[junit4] 2> NOTE: All tests run in this JVM: [CurrencyFieldOpenExchangeTest, HdfsChaosMonkeySafeLeaderTest, LeaderElectionTest, ReturnFieldsTest, DistributedQueryComponentOptimizationTest, JsonLoaderTest, AnalyticsQueryTest, HdfsSyncSliceTest, ZkControllerTest, BasicDistributedZkTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestArbitraryIndexDir, TestFieldSortValues, XsltUpdateRequestHandlerTest, TestSort, DistribCursorPagingTest, ZkNodePropsTest, NoCacheHeaderTest, FileUtilsTest, TestStressVersions, TestStressRecovery, SharedFSAutoReplicaFailoverUtilsTest, TestExceedMaxTermLength, TestAnalyzeInfixSuggestions, TestCloudSchemaless, AnalyticsMergeStrategyTest, TestAnalyzedSuggestions, UpdateParamsTest, TestConfig, TestSolr4Spatial2, TermsComponentTest, SignatureUpdateProcessorFactoryTest, TestCollectionAPI, TestSolrCoreProperties, BlockCacheTest, FastVectorHighlighterTest, PrimitiveFieldTypeTest, RecoveryZkTest, JSONWriterTest, BasicFunctionalityTest, LeaderFailoverAfterPartitionTest, TestSearchPerf, TestQuerySenderListener, IndexBasedSpellCheckerTest, BadComponentTest, SSLMigrationTest, TestPivotHelperCode, DistributedFacetPivotLongTailTest, DocumentAnalysisRequestHandlerTest, FieldMutatingUpdateProcessorTest, TestHighlightDedupGrouping, AsyncMigrateRouteKeyTest, ZkCLITest, TestLMJelinekMercerSimilarityFactory, TestDefaultSimilarityFactory, TestSchemaVersionResource, TestHighFrequencyDictionaryFactory, TestCursorMarkWithoutUniqueKey, TestLuceneMatchVersion, TestFuzzyAnalyzedSuggestions, DocValuesMissingTest, ChangedSchemaMergeTest, TestComponentsName, TestCopyFieldCollectionResource, TestStandardQParsers, DistributedFacetPivotLargeTest, AddBlockUpdateTest, MigrateRouteKeyTest, TestJmxMonitoredMap, TestSolrDeletionPolicy2, ParsingFieldUpdateProcessorsTest, TestReload, TestSolrDeletionPolicy1, TestCoreDiscovery, HdfsBasicDistributedZk2Test, TestInfoStreamLogging, HdfsCollectionsAPIDistributedZkTest, TestSolrQueryParserResource, TestFoldingMultitermQuery, TestTrie, DocValuesMultiTest, TestSchemaNameResource, MinimalSchemaTest, TestStressReorder, TestCodecSupport, QueryEqualityTest, UnloadDistributedZkTest, TestSurroundQueryParser, RequiredFieldsTest, TestCloudManagedSchemaConcurrent, ExternalFileFieldSortTest, TestSuggestSpellingConverter, TestRestManager, InfoHandlerTest, TestSchemaSimilarityResource, HttpPartitionTest, TestJmxIntegration, TestManagedResourceStorage, TestSolrXmlPersistor, TestRangeQuery, SpellingQueryConverterTest, SolrIndexSplitterTest, CoreMergeIndexesAdminHandlerTest, CoreAdminHandlerTest, BasicDistributedZk2Test, SyncSliceTest, OverseerTest]
[junit4] Completed on J0 in 63.93s, 9 tests, 1 error <<< FAILURES!
[...truncated 767 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/build.xml:474: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/build.xml:454: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/solr/common-build.xml:494: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/lucene/common-build.xml:1351: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/lucene/common-build.xml:966: There were test failures: 424 suites, 1762 tests, 1 error, 39 ignored (18 assumptions)
Total time: 99 minutes 55 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 64bit/jdk1.8.0_31 -XX:+UseCompressedOops -XX:+UseSerialGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any