You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2015/12/24 10:49:09 UTC
[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 3871 - Failure
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/3871/
1 tests failed.
FAILED: org.apache.solr.cloud.OverseerTest.testExternalClusterStateChangeBehavior
Error Message:
Illegal state, was: down expected:active clusterState:live nodes:[]collections:{c1=DocCollection(c1)={ "shards":{"shard1":{ "range":null, "state":"active", "parent":null, "replicas":{"core_node1":{ "base_url":"http://127.0.0.1/solr", "node_name":"node1", "core":"core1", "roles":"", "state":"down"}}}}, "router":{"name":"implicit"}}, test=LazyCollectionRef(test)}
Stack Trace:
java.lang.AssertionError: Illegal state, was: down expected:active clusterState:live nodes:[]collections:{c1=DocCollection(c1)={
"shards":{"shard1":{
"range":null,
"state":"active",
"parent":null,
"replicas":{"core_node1":{
"base_url":"http://127.0.0.1/solr",
"node_name":"node1",
"core":"core1",
"roles":"",
"state":"down"}}}},
"router":{"name":"implicit"}}, test=LazyCollectionRef(test)}
at __randomizedtesting.SeedInfo.seed([C12162509DF0D25A:A93F61BC7F608814]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractDistribZkTestBase.verifyReplicaStatus(AbstractDistribZkTestBase.java:237)
at org.apache.solr.cloud.OverseerTest.testExternalClusterStateChangeBehavior(OverseerTest.java:1262)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 10321 lines...]
[junit4] Suite: org.apache.solr.cloud.OverseerTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.OverseerTest_C12162509DF0D25A-001/init-core-data-001
[junit4] 2> 486301 INFO (SUITE-OverseerTest-seed#[C12162509DF0D25A]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false)
[junit4] 2> 486301 INFO (SUITE-OverseerTest-seed#[C12162509DF0D25A]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore
[junit4] 2> 486301 INFO (SUITE-OverseerTest-seed#[C12162509DF0D25A]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore end
[junit4] 2> 486306 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testOverseerStatsReset
[junit4] 2> 486306 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 486306 INFO (Thread-3932) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 486307 INFO (Thread-3932) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 486406 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ZkTestServer start zk server on port:46571
[junit4] 2> 486407 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 486407 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 486410 INFO (zkCallback-597-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4953f0fe name:ZooKeeperConnection Watcher:127.0.0.1:46571 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 486410 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 486410 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 486412 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 486412 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 486413 INFO (zkCallback-598-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@760e7c17 name:ZooKeeperConnection Watcher:127.0.0.1:46571 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 486413 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 486414 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 486414 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr
[junit4] 2> 486416 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 486416 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 486417 INFO (zkCallback-599-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7ca13683 name:ZooKeeperConnection Watcher:127.0.0.1:46571/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 486418 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 486418 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 486418 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
[junit4] 2> 486419 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections
[junit4] 2> 486421 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
[junit4] 2> 486422 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
[junit4] 2> 486423 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /security.json
[junit4] 2> 486424 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 486427 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 486427 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 486429 INFO (zkCallback-600-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@50fc3ac1 name:ZooKeeperConnection Watcher:127.0.0.1:46571/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 486429 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 486430 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 486431 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 486433 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/node1
[junit4] 2> 486434 INFO (zkCallback-599-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 486435 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
[junit4] 2> 486436 INFO (zkCallback-600-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 486436 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
[junit4] 2> 486438 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
[junit4] 2> 486439 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 486440 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95089375675351042-127.0.0.1:46571_solr-n_0000000000
[junit4] 2> 486440 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46571_solr
[junit4] 2> 486440 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
[junit4] 2> 486441 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.Overseer Overseer (id=95089375675351042-127.0.0.1:46571_solr-n_0000000000) starting
[junit4] 2> 486442 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
[junit4] 2> 486445 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
[junit4] 2> 486446 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
[junit4] 2> 486448 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
[junit4] 2> 486450 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
[junit4] 2> 486452 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
[junit4] 2> 486455 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 486455 INFO (OverseerCollectionConfigSetProcessor-95089375675351042-127.0.0.1:46571_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 486456 INFO (OverseerStateUpdate-95089375675351042-127.0.0.1:46571_solr-n_0000000000) [ ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 486459 INFO (OverseerStateUpdate-95089375675351042-127.0.0.1:46571_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"} current state version: 0
[junit4] 2> 486459 INFO (OverseerStateUpdate-95089375675351042-127.0.0.1:46571_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 486459 INFO (OverseerStateUpdate-95089375675351042-127.0.0.1:46571_solr-n_0000000000) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
[junit4] 2> 486459 INFO (OverseerStateUpdate-95089375675351042-127.0.0.1:46571_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 486460 INFO (zkCallback-599-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 486460 INFO (zkCallback-600-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 486958 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 486960 INFO (zkCallback-599-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
[junit4] 2> 486960 INFO (zkCallback-600-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
[junit4] 2> 486964 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard1/election/95089375675351043-node1_core1-n_0000000000
[junit4] 2> 486965 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leaders/shard1
[junit4] 2> 486967 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node
[junit4] 2> 486970 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.Overseer Overseer (id=95089375675351042-127.0.0.1:46571_solr-n_0000000000) closing
[junit4] 2> 486970 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ElectionContext Canceling election /overseer_elect/election/95089375675351042-127.0.0.1:46571_solr-n_0000000000
[junit4] 2> 486971 INFO (OverseerStateUpdate-95089375675351042-127.0.0.1:46571_solr-n_0000000000) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:46571_solr
[junit4] 2> 486977 INFO (OverseerExitThread) [ ] o.a.s.c.Overseer I'm exiting , but I'm still the leader
[junit4] 2> 486977 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95089375675351042-127.0.0.1:46571_solr-n_0000000001
[junit4] 2> 486978 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46571_solr
[junit4] 2> 486978 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
[junit4] 2> 486979 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.Overseer Overseer (id=95089375675351042-127.0.0.1:46571_solr-n_0000000001) starting
[junit4] 2> 486987 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 486987 INFO (OverseerCollectionConfigSetProcessor-95089375675351042-127.0.0.1:46571_solr-n_0000000001) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 486988 INFO (zkCallback-599-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 486988 INFO (zkCallback-600-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 486988 INFO (OverseerStateUpdate-95089375675351042-127.0.0.1:46571_solr-n_0000000001) [ ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 486995 INFO (OverseerStateUpdate-95089375675351042-127.0.0.1:46571_solr-n_0000000001) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "base_url":"http://node1/solr/",
[junit4] 2> "core":"core1",
[junit4] 2> "state":"active"} current state version: 1
[junit4] 2> 486998 INFO (zkCallback-599-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 488488 WARN (zkCallback-600-thread-1) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 488491 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46571 46571
[junit4] 2> 488715 INFO (Thread-3932) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46571 46571
[junit4] 2> 488716 WARN (Thread-3932) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/aliases.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2> 2 /solr/overseer/collection-queue-work
[junit4] 2> 2 /solr/collections
[junit4] 2> 2 /solr/live_nodes
[junit4] 2>
[junit4] 2> 488716 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testOverseerStatsReset
[junit4] 2> 488716 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C12162509DF0D25A]) [ ] o.a.s.c.Overseer Overseer (id=95089375675351042-127.0.0.1:46571_solr-n_0000000001) closing
[junit4] 2> 488717 INFO (OverseerStateUpdate-95089375675351042-127.0.0.1:46571_solr-n_0000000001) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:46571_solr
[junit4] 2> 488726 ERROR (OverseerExitThread) [ ] o.a.s.c.Overseer could not read the data
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:342)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:305)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:88)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$2.run(Overseer.java:266)
[junit4] 2> 488732 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDoubleAssignment
[junit4] 2> 488732 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 488733 INFO (Thread-3933) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 488733 INFO (Thread-3933) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 488833 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ZkTestServer start zk server on port:44352
[junit4] 2> 488833 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 488833 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 488836 INFO (zkCallback-602-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6e41a9ae name:ZooKeeperConnection Watcher:127.0.0.1:44352/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 488836 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 488836 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 488836 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 488836 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 488838 INFO (zkCallback-603-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@75ab935b name:ZooKeeperConnection Watcher:127.0.0.1:44352 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 488838 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 488838 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 488839 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 488839 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 488840 INFO (zkCallback-604-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@207a182 name:ZooKeeperConnection Watcher:127.0.0.1:44352 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 488841 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 488841 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 488841 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr
[junit4] 2> 488844 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
[junit4] 2> 488845 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections
[junit4] 2> 488846 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
[junit4] 2> 488847 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
[junit4] 2> 488848 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /security.json
[junit4] 2> 488849 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 488851 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 488851 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 488856 INFO (zkCallback-605-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7dcdcc60 name:ZooKeeperConnection Watcher:127.0.0.1:44352/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 488856 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 488856 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 488858 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 488860 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/node1
[junit4] 2> 488861 INFO (zkCallback-602-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 488861 INFO (zkCallback-605-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 488862 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 488862 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 488863 INFO (zkCallback-606-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@12073db7 name:ZooKeeperConnection Watcher:127.0.0.1:44352/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 488864 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 488864 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 488864 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
[junit4] 2> 488865 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
[junit4] 2> 488866 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
[junit4] 2> 488868 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 488868 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95089375834341380-127.0.0.1:44352_solr-n_0000000000
[junit4] 2> 488868 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:44352_solr
[junit4] 2> 488869 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
[junit4] 2> 488870 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.Overseer Overseer (id=95089375834341380-127.0.0.1:44352_solr-n_0000000000) starting
[junit4] 2> 488871 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
[junit4] 2> 488873 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
[junit4] 2> 488875 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
[junit4] 2> 488877 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
[junit4] 2> 488879 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
[junit4] 2> 488881 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
[junit4] 2> 488885 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 488885 INFO (OverseerCollectionConfigSetProcessor-95089375834341380-127.0.0.1:44352_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 488886 INFO (OverseerStateUpdate-95089375834341380-127.0.0.1:44352_solr-n_0000000000) [ ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 488886 INFO (OverseerStateUpdate-95089375834341380-127.0.0.1:44352_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 488890 INFO (OverseerStateUpdate-95089375834341380-127.0.0.1:44352_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"} current state version: 0
[junit4] 2> 488890 INFO (OverseerStateUpdate-95089375834341380-127.0.0.1:44352_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 488890 INFO (OverseerStateUpdate-95089375834341380-127.0.0.1:44352_solr-n_0000000000) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
[junit4] 2> 488890 INFO (OverseerStateUpdate-95089375834341380-127.0.0.1:44352_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 488892 INFO (zkCallback-602-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 488892 INFO (zkCallback-605-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 488892 INFO (zkCallback-606-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 489388 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 489392 INFO (zkCallback-605-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
[junit4] 2> 489392 INFO (zkCallback-606-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
[junit4] 2> 489392 INFO (zkCallback-602-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
[junit4] 2> 489393 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard1/election/95089375834341379-node1_core1-n_0000000000
[junit4] 2> 489394 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leaders/shard1
[junit4] 2> 489396 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node
[junit4] 2> 489399 INFO (OverseerStateUpdate-95089375834341380-127.0.0.1:44352_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "base_url":"http://node1/solr/",
[junit4] 2> "core":"core1",
[junit4] 2> "state":"active"} current state version: 1
[junit4] 2> 489400 INFO (zkCallback-602-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 489400 INFO (zkCallback-605-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 489400 INFO (zkCallback-606-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 489502 INFO (zkCallback-606-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 489502 INFO (zkCallback-602-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 490900 WARN (zkCallback-605-thread-1) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 490901 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 490914 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 490920 INFO (zkCallback-608-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7f75fc2d name:ZooKeeperConnection Watcher:127.0.0.1:44352/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 490920 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 490920 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 490922 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 490923 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/node1
[junit4] 2> 490925 INFO (zkCallback-606-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 490925 INFO (zkCallback-602-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 490926 INFO (zkCallback-608-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 490928 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 490928 INFO (OverseerStateUpdate-95089375834341380-127.0.0.1:44352_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"} current state version: 2
[junit4] 2> 490929 INFO (OverseerStateUpdate-95089375834341380-127.0.0.1:44352_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 490929 INFO (OverseerStateUpdate-95089375834341380-127.0.0.1:44352_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard1 is already registered
[junit4] 2> 490931 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard1/election/95089375834341381-node1_core1-n_0000000001
[junit4] 2> 490932 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node
[junit4] 2> 490934 INFO (OverseerStateUpdate-95089375834341380-127.0.0.1:44352_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "base_url":"http://node1/solr/",
[junit4] 2> "core":"core1",
[junit4] 2> "state":"active"} current state version: 2
[junit4] 2> 491037 INFO (zkCallback-606-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 491037 INFO (zkCallback-602-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 491040 INFO (zkCallback-602-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 491041 INFO (zkCallback-608-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 491041 INFO (zkCallback-608-thread-2) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 491041 WARN (zkCallback-608-thread-2) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 491043 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x151d34894850005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 492539 WARN (OverseerStateUpdate-95089375834341380-127.0.0.1:44352_solr-n_0000000000) [ ] o.a.s.c.Overseer Solr cannot talk to ZK, exiting Overseer main queue loop
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/queue-work
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:328)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:325)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:325)
[junit4] 2> at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:312)
[junit4] 2> at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:291)
[junit4] 2> at org.apache.solr.cloud.DistributedQueue.removeFirst(DistributedQueue.java:380)
[junit4] 2> at org.apache.solr.cloud.DistributedQueue.poll(DistributedQueue.java:189)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:240)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 492540 INFO (OverseerStateUpdate-95089375834341380-127.0.0.1:44352_solr-n_0000000000) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:44352_solr
[junit4] 2> 492547 ERROR (OverseerExitThread) [ ] o.a.s.c.Overseer could not read the data
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:342)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:305)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:88)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$2.run(Overseer.java:266)
[junit4] 2> 492547 WARN (zkCallback-608-thread-1) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /clusterstate.json]
[junit4] 2> 492550 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44352 44352
[junit4] 2> 492759 INFO (Thread-3933) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44352 44352
[junit4] 2> 492760 WARN (Thread-3933) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/aliases.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/collections
[junit4] 2> 3 /solr/live_nodes
[junit4] 2>
[junit4] 2> 492761 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDoubleAssignment
[junit4] 2> 492761 INFO (TEST-OverseerTest.testDoubleAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.Overseer Overseer (id=95089375834341380-127.0.0.1:44352_solr-n_0000000000) closing
[junit4] 2> 492766 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testExternalClusterStateChangeBehavior
[junit4] 2> 492767 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 492776 INFO (Thread-3934) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 492776 INFO (Thread-3934) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 492873 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ZkTestServer start zk server on port:50217
[junit4] 2> 492873 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 492876 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 492883 INFO (zkCallback-609-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@478fa24d name:ZooKeeperConnection Watcher:127.0.0.1:50217/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 492883 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 492884 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 492884 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 492885 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 492897 INFO (zkCallback-610-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4f66273e name:ZooKeeperConnection Watcher:127.0.0.1:50217 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 492897 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 492897 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 492898 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 492900 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 492906 INFO (zkCallback-611-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4a7668c name:ZooKeeperConnection Watcher:127.0.0.1:50217 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 492906 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 492907 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 492907 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr
[junit4] 2> 492920 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
[junit4] 2> 492921 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections
[junit4] 2> 492922 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
[junit4] 2> 492923 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
[junit4] 2> 492924 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /security.json
[junit4] 2> 492926 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 492927 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 492930 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 492930 INFO (zkCallback-612-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@52334b9 name:ZooKeeperConnection Watcher:127.0.0.1:50217/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 492931 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 492931 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 492931 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
[junit4] 2> 492932 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
[junit4] 2> 492933 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
[junit4] 2> 492934 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 492935 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95089376099434499-127.0.0.1:50217_solr-n_0000000000
[junit4] 2> 492935 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:50217_solr
[junit4] 2> 492935 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
[junit4] 2> 492936 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.Overseer Overseer (id=95089376099434499-127.0.0.1:50217_solr-n_0000000000) starting
[junit4] 2> 492937 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
[junit4] 2> 492939 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
[junit4] 2> 492940 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
[junit4] 2> 492942 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
[junit4] 2> 492944 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
[junit4] 2> 492945 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
[junit4] 2> 492948 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 492949 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 492950 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 492951 INFO (OverseerCollectionConfigSetProcessor-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 492955 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"down"} current state version: 0
[junit4] 2> 492955 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"down"}
[junit4] 2> 492955 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 492956 INFO (zkCallback-612-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 492957 INFO (zkCallback-609-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 493008 INFO (zkCallback-609-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 493008 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"} current state version: 1
[junit4] 2> 493008 INFO (zkCallback-612-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 493010 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 493010 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator node=core_node1 is already registered
[junit4] 2> 493010 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard1 is already registered
[junit4] 2> 493015 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 4, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"} current state version: 1
[junit4] 2> 493015 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"}
[junit4] 2> 493015 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator node=core_node1 is already registered
[junit4] 2> 493015 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard1 is already registered
[junit4] 2> 493017 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 4, message = {
[junit4] 2> "operation":"create",
[junit4] 2> "name":"test",
[junit4] 2> "numShards":"1",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "stateFormat":"2"} current state version: 1
[junit4] 2> 493017 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: test
[junit4] 2> 493018 WARN (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.Overseer Bad version writing to ZK using compare-and-set, will force refresh cluster state
[junit4] 2> org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion for /clusterstate.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:115)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.setData(ZooKeeper.java:1270)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:362)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:359)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.setData(SolrZkClient.java:359)
[junit4] 2> at org.apache.solr.cloud.overseer.ZkStateWriter.writePendingUpdates(ZkStateWriter.java:253)
[junit4] 2> at org.apache.solr.cloud.overseer.ZkStateWriter.enqueueUpdate(ZkStateWriter.java:120)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.processQueueItem(Overseer.java:291)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:215)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 493020 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 4, message = {
[junit4] 2> "operation":"create",
[junit4] 2> "name":"test",
[junit4] 2> "numShards":"1",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "stateFormat":"2"} current state version: 2
[junit4] 2> 493020 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: test
[junit4] 2> 493021 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/test/state.json
[junit4] 2> 493023 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 4, message = {
[junit4] 2> "operation":"createshard",
[junit4] 2> "collection":"test",
[junit4] 2> "shard":"x",
[junit4] 2> "replicationFactor":"1"} current state version: 2
[junit4] 2> 493024 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 4, message = {
[junit4] 2> "operation":"addreplica",
[junit4] 2> "collection":"test",
[junit4] 2> "shard":"x",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "state":"down"} current state version: 2
[junit4] 2> 493024 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"addreplica",
[junit4] 2> "collection":"test",
[junit4] 2> "shard":"x",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "state":"down"}
[junit4] 2> 493131 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/test/state.json version: 0
[junit4] 2> 498182 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50217 50217
[junit4] 2> 498348 INFO (Thread-3934) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50217 50217
[junit4] 2> 498349 WARN (Thread-3934) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/aliases.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/collections
[junit4] 2> 2 /solr/live_nodes
[junit4] 2>
[junit4] 2> 498350 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testExternalClusterStateChangeBehavior
[junit4] 2> 498350 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C12162509DF0D25A]) [ ] o.a.s.c.Overseer Overseer (id=95089376099434499-127.0.0.1:50217_solr-n_0000000000) closing
[junit4] 2> 498350 INFO (OverseerStateUpdate-95089376099434499-127.0.0.1:50217_solr-n_0000000000) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:50217_solr
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=OverseerTest -Dtests.method=testExternalClusterStateChangeBehavior -Dtests.seed=C12162509DF0D25A -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=pl -Dtests.timezone=Europe/Busingen -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] FAILURE 5.59s J2 | OverseerTest.testExternalClusterStateChangeBehavior <<<
[junit4] > Throwable #1: java.lang.AssertionError: Illegal state, was: down expected:active clusterState:live nodes:[]collections:{c1=DocCollection(c1)={
[junit4] > "shards":{"shard1":{
[junit4] > "range":null,
[junit4] > "state":"active",
[junit4] > "parent":null,
[junit4] > "replicas":{"core_node1":{
[junit4] > "base_url":"http://127.0.0.1/solr",
[junit4] > "node_name":"node1",
[junit4] > "core":"core1",
[junit4] > "roles":"",
[junit4] > "state":"down"}}}},
[junit4] > "router":{"name":"implicit"}}, test=LazyCollectionRef(test)}
[junit4] > at __randomizedtesting.SeedInfo.seed([C12162509DF0D25A:A93F61BC7F608814]:0)
[junit4] > at org.apache.solr.cloud.AbstractDistribZkTestBase.verifyReplicaStatus(AbstractDistribZkTestBase.java:237)
[junit4] > at org.apache.solr.cloud.OverseerTest.testExternalClusterStateChangeBehavior(OverseerTest.java:1262)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 498356 ERROR (OverseerExitThread) [ ] o.a.s.c.Overseer could not read the data
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:342)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:305)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:88)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$2.run(Overseer.java:266)
[junit4] 2> 498358 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testShardAssignment
[junit4] 2> 498358 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 498359 INFO (Thread-3935) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 498359 INFO (Thread-3935) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 498459 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ZkTestServer start zk server on port:48446
[junit4] 2> 498459 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 498459 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 498461 INFO (zkCallback-614-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@50966248 name:ZooKeeperConnection Watcher:127.0.0.1:48446 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 498462 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 498462 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 498463 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 498463 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x151d348ba210000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 498463 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 498465 INFO (zkCallback-615-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@63ff97f4 name:ZooKeeperConnection Watcher:127.0.0.1:48446 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 498465 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 498465 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 498465 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr
[junit4] 2> 498467 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 498468 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 498469 INFO (zkCallback-616-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@64e4acbe name:ZooKeeperConnection Watcher:127.0.0.1:48446/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 498469 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 498469 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 498470 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
[junit4] 2> 498471 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections
[junit4] 2> 498472 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
[junit4] 2> 498474 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
[junit4] 2> 498475 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /security.json
[junit4] 2> 498476 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 498477 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 498478 INFO (zkCallback-617-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2108fba name:ZooKeeperConnection Watcher:127.0.0.1:48446/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 498478 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 498478 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 498478 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
[junit4] 2> 498479 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
[junit4] 2> 498480 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
[junit4] 2> 498482 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 498482 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95089376465321987-127.0.0.1:48446_solr-n_0000000000
[junit4] 2> 498483 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:48446_solr
[junit4] 2> 498483 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
[junit4] 2> 498484 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.Overseer Overseer (id=95089376465321987-127.0.0.1:48446_solr-n_0000000000) starting
[junit4] 2> 498485 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
[junit4] 2> 498487 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
[junit4] 2> 498489 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
[junit4] 2> 498491 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
[junit4] 2> 498493 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
[junit4] 2> 498495 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
[junit4] 2> 498498 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 498499 INFO (OverseerCollectionConfigSetProcessor-95089376465321987-127.0.0.1:48446_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 498499 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 498499 INFO (OverseerStateUpdate-95089376465321987-127.0.0.1:48446_solr-n_0000000000) [ ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 498500 INFO (OverseerStateUpdate-95089376465321987-127.0.0.1:48446_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 498502 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 498503 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 498508 INFO (zkCallback-619-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2d3db267 name:ZooKeeperConnection Watcher:127.0.0.1:48446/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 498508 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 498508 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 498510 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 498511 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1
[junit4] 2> 498513 INFO (zkCallback-617-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 498513 INFO (zkCallback-616-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 498513 INFO (zkCallback-619-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 498517 INFO (OverseerStateUpdate-95089376465321987-127.0.0.1:48446_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"} current state version: 0
[junit4] 2> 498518 INFO (OverseerStateUpdate-95089376465321987-127.0.0.1:48446_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 498518 INFO (OverseerStateUpdate-95089376465321987-127.0.0.1:48446_solr-n_0000000000) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
[junit4] 2> 498518 INFO (OverseerStateUpdate-95089376465321987-127.0.0.1:48446_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 498519 INFO (zkCallback-616-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 498519 INFO (zkCallback-617-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 498520 INFO (zkCallback-619-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 499017 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 499018 INFO (zkCallback-617-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
[junit4] 2> 499018 INFO (zkCallback-616-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
[junit4] 2> 499018 INFO (zkCallback-619-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
[junit4] 2> 499022 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard1/election/95089376465321988-127.0.0.1_core1-n_0000000000
[junit4] 2> 499023 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leaders/shard1
[junit4] 2> 499025 INFO (TEST-OverseerTest.testShardAssignment-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node
[junit4] 2> 499028 INFO (OverseerStateUpdate-95089376465321987-127.0.0.1:48446_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "base_url":"http://127.0.0.1/solr/",
[junit4] 2> "core":"core1",
[junit4] 2> "state":"active"} current state version: 1
[junit4] 2> 499030 INFO (OverseerStateUpdate-95089376465321987-127.0.0.1:48446_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core2",
[junit4] 2> "core_node_name":"node2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"} current state version: 1
[junit4] 2> 499030 INFO (OverseerStateUpdate-95089376465321987-127.0.0.1:48446_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core2",
[junit4] 2> "core_node_name":"node2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 499030 INFO (OverseerStateUpdate-95089376465321987-127.0.0.1:48446_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Collection already exists with numShards=3
[junit4] 2> 499030 INFO (OverseerStateUpdate-9508937
[...truncated too long message...]
itAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 537310 INFO (OverseerCollectionConfigSetProcessor-95089379006808067-127.0.0.1:36676_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 537311 INFO (OverseerStateUpdate-95089379006808067-127.0.0.1:36676_solr-n_0000000000) [ ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 537312 INFO (OverseerStateUpdate-95089379006808067-127.0.0.1:36676_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 537316 INFO (OverseerStateUpdate-95089379006808067-127.0.0.1:36676_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"} current state version: 0
[junit4] 2> 537317 INFO (OverseerStateUpdate-95089379006808067-127.0.0.1:36676_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 537317 INFO (OverseerStateUpdate-95089379006808067-127.0.0.1:36676_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 537319 INFO (zkCallback-700-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 537319 INFO (zkCallback-703-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 537368 INFO (OverseerStateUpdate-95089379006808067-127.0.0.1:36676_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"} current state version: 1
[junit4] 2> 537369 INFO (OverseerStateUpdate-95089379006808067-127.0.0.1:36676_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"}
[junit4] 2> 537369 INFO (OverseerStateUpdate-95089379006808067-127.0.0.1:36676_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator node=core_node1 is already registered
[junit4] 2> 537369 INFO (OverseerStateUpdate-95089379006808067-127.0.0.1:36676_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard1 is already registered
[junit4] 2> 537473 INFO (zkCallback-700-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 537473 INFO (zkCallback-703-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 537522 INFO (TEST-OverseerTest.testStateChange-seed#[C12162509DF0D25A]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36676 36676
[junit4] 2> 537768 INFO (Thread-3943) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36676 36676
[junit4] 2> 537769 WARN (Thread-3943) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/aliases.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/collections
[junit4] 2> 2 /solr/live_nodes
[junit4] 2>
[junit4] 2> 537770 INFO (TEST-OverseerTest.testStateChange-seed#[C12162509DF0D25A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testStateChange
[junit4] 2> 537770 INFO (TEST-OverseerTest.testStateChange-seed#[C12162509DF0D25A]) [ ] o.a.s.c.Overseer Overseer (id=95089379006808067-127.0.0.1:36676_solr-n_0000000000) closing
[junit4] 2> 537770 INFO (OverseerStateUpdate-95089379006808067-127.0.0.1:36676_solr-n_0000000000) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:36676_solr
[junit4] 2> 537772 ERROR (OverseerExitThread) [ ] o.a.s.c.Overseer could not read the data
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:342)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:305)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:88)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$2.run(Overseer.java:266)
[junit4] 2> 540773 INFO (SUITE-OverseerTest-seed#[C12162509DF0D25A]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.OverseerTest_C12162509DF0D25A-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene54): {}, docValues:{}, sim=DefaultSimilarity, locale=pl, timezone=Europe/Busingen
[junit4] 2> NOTE: Linux 3.13.0-52-generic amd64/Oracle Corporation 1.7.0_80 (64-bit)/cpus=4,threads=1,free=35985984,total=396361728
[junit4] 2> NOTE: All tests run in this JVM: [SuggesterWFSTTest, SparseHLLTest, TestAnalyzeInfixSuggestions, TestHashQParserPlugin, TestLeaderInitiatedRecoveryThread, TestCustomDocTransformer, XmlUpdateRequestHandlerTest, TestSweetSpotSimilarityFactory, TestIndexSearcher, TestCollationFieldDocValues, TestRecoveryHdfs, TestNamedUpdateProcessors, RegexBoostProcessorTest, ClusterStateTest, SliceStateTest, AnalyticsMergeStrategyTest, MergeStrategyTest, TestWriterPerf, StatelessScriptUpdateProcessorFactoryTest, TestIntervalFaceting, TestScoreJoinQPScore, TestCoreDiscovery, HdfsLockFactoryTest, BadIndexSchemaTest, HdfsBasicDistributedZkTest, SolrTestCaseJ4Test, TestCollationField, TestRemoveLastDynamicCopyField, TestReload, TestSolrConfigHandlerConcurrent, TestSolrIndexConfig, TestClassNameShortening, SampleTest, TestSolrJ, TestFreeTextSuggestions, CurrencyFieldXmlFileTest, TestSolrQueryResponse, CleanupOldIndexTest, CoreAdminHandlerTest, BlockDirectoryTest, TestRequestStatusCollectionAPI, TestDistribIDF, TestElisionMultitermQuery, FieldAnalysisRequestHandlerTest, AutoCommitTest, FullHLLTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestTrackingShardHandlerFactory, CoreMergeIndexesAdminHandlerTest, TestLRUStatsCache, TestIndexingPerformance, RecoveryZkTest, DeleteReplicaTest, TestGroupingSearch, TestSolrQueryParserDefaultOperatorResource, DocValuesMultiTest, HardAutoCommitTest, TestManagedStopFilterFactory, ScriptEngineTest, OverseerTest]
[junit4] Completed [172/555 (1!)] on J2 in 54.50s, 12 tests, 1 failure, 1 skipped <<< FAILURES!
[...truncated 1225 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:794: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:738: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:233: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:526: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1456: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1010: There were test failures: 555 suites (7 ignored), 2209 tests, 1 failure, 53 ignored (38 assumptions) [seed: C12162509DF0D25A]
Total time: 63 minutes 7 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any