You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/03/21 17:02:01 UTC

[JENKINS] Lucene-Solr-4.x-Linux (64bit/jdk1.7.0_51) - Build # 9760 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/9760/
Java: 64bit/jdk1.7.0_51 -XX:+UseCompressedOops -XX:+UseG1GC -XX:-UseSuperWord

1 tests failed.
FAILED:  org.apache.solr.client.solrj.impl.CloudSolrServerTest.testShutdown

Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:53377 within 45000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:53377 within 45000 ms
	at __randomizedtesting.SeedInfo.seed([94AB6F788676E001:77DD66ED610C7573]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:150)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:101)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:91)
	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:91)
	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:85)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.setUp(AbstractDistribZkTestBase.java:75)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.setUp(AbstractFullDistribZkTestBase.java:201)
	at org.apache.solr.client.solrj.impl.CloudSolrServerTest.setUp(CloudSolrServerTest.java:93)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1617)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:860)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:876)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:359)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:783)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:443)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:835)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:771)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:782)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:359)
	at java.lang.Thread.run(Thread.java:744)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:53377 within 45000 ms
	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:223)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:142)
	... 45 more




Build Log:
[...truncated 11668 lines...]
   [junit4] Suite: org.apache.solr.client.solrj.impl.CloudSolrServerTest
   [junit4]   2> 20621 T117 oas.SolrTestCaseJ4.startTrackingSearchers WARN startTrackingSearchers: numOpens=7 numCloses=7
   [junit4]   2> 20622 T117 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 20622 T117 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 20624 T117 oasc.AbstractZkTestCase.<clinit> WARN TEST_HOME() does not exist - solrj test?
   [junit4]   2> 20628 T117 oas.SolrTestCaseJ4.setUp ###Starting testShutdown
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./solrtest-CloudSolrServerTest-1395417592011
   [junit4]   2> 20631 T117 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 20633 T118 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 20732 T117 oasc.ZkTestServer.run start zk server on port:53377
   [junit4]   2> 20765 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 68202 T121 oazsp.FileTxnLog.commit WARN fsync-ing the write ahead log in SyncThread:0 took 47421ms which will adversely effect operation latency. See the ZooKeeper troubleshooting guide
   [junit4]   2> 68203 T117 oas.SolrTestCaseJ4.tearDown ###Ending testShutdown
   [junit4]   2> 68217 T119 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x144e55eb0fb0000, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 68218 T119 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x144e55eb0fb0001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 68219 T119 oazs.NIOServerCnxnFactory.run WARN Ignoring unexpected runtime exception java.nio.channels.CancelledKeyException
   [junit4]   2> 	at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
   [junit4]   2> 	at sun.nio.ch.SelectionKeyImpl.readyOps(SelectionKeyImpl.java:87)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:187)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 68220 T119 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x144e55eb0fb0003, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 68221 T117 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:53377 53377
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CloudSolrServerTest -Dtests.method=testShutdown -Dtests.seed=94AB6F788676E001 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es_PE -Dtests.timezone=America/New_York -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   47.8s J0 | CloudSolrServerTest.testShutdown <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:53377 within 45000 ms
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([94AB6F788676E001:77DD66ED610C7573]:0)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:150)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:101)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:91)
   [junit4]    > 	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:91)
   [junit4]    > 	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:85)
   [junit4]    > 	at org.apache.solr.cloud.AbstractDistribZkTestBase.setUp(AbstractDistribZkTestBase.java:75)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.setUp(AbstractFullDistribZkTestBase.java:201)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServerTest.setUp(CloudSolrServerTest.java:93)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:744)
   [junit4]    > Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:53377 within 45000 ms
   [junit4]    > 	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:223)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:142)
   [junit4]    > 	... 45 more
   [junit4]   2> 68393 T117 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./solrtest-CloudSolrServerTest-1395417639775
   [junit4]   2> 68393 T117 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 68394 T125 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 68494 T117 oasc.ZkTestServer.run start zk server on port:58441
   [junit4]   2> 68495 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 68531 T131 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@799b0941 name:ZooKeeperConnection Watcher:127.0.0.1:58441 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 68532 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 68534 T117 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 68558 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 68560 T133 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@245a1037 name:ZooKeeperConnection Watcher:127.0.0.1:58441/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 68560 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 68566 T117 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 68571 T117 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 68576 T117 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 68579 T117 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 68583 T117 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 68584 T117 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 68590 T117 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 68591 T117 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 68595 T117 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml because it doesn't exist
   [junit4]   2> 68596 T117 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/stopwords.txt because it doesn't exist
   [junit4]   2> 68596 T117 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/protwords.txt because it doesn't exist
   [junit4]   2> 68597 T117 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/currency.xml because it doesn't exist
   [junit4]   2> 68597 T117 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/enumsConfig.xml because it doesn't exist
   [junit4]   2> 68598 T117 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/open-exchange-rates.json because it doesn't exist
   [junit4]   2> 68598 T117 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/mapping-ISOLatin1Accent.txt because it doesn't exist
   [junit4]   2> 68599 T117 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/old_synonyms.txt because it doesn't exist
   [junit4]   2> 68599 T117 oasc.AbstractZkTestCase.putConfig skipping /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/solrj/src/test-files/solrj/solr/collection1/conf/synonyms.txt because it doesn't exist
   [junit4]   2> 68606 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 68608 T135 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@362239c7 name:ZooKeeperConnection Watcher:127.0.0.1:58441/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 68609 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 68653 T117 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 68659 T117 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 68669 T117 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:57950
   [junit4]   2> 68671 T117 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 68672 T117 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 68672 T117 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1395417639993
   [junit4]   2> 68672 T117 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1395417639993/'
   [junit4]   2> 68707 T117 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1395417639993/solr.xml
   [junit4]   2> 68759 T117 oasc.CoreContainer.<init> New CoreContainer 304496177
   [junit4]   2> 68759 T117 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1395417639993/]
   [junit4]   2> 68760 T117 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 68761 T117 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 68761 T117 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 68761 T117 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 68762 T117 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 68762 T117 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 68762 T117 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 68763 T117 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 68763 T117 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 68767 T117 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 68768 T117 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 68768 T117 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 68769 T117 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58441/solr
   [junit4]   2> 68782 T117 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 68783 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 68787 T146 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c4ba63b name:ZooKeeperConnection Watcher:127.0.0.1:58441 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 68788 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 68793 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 68795 T148 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@211a5528 name:ZooKeeperConnection Watcher:127.0.0.1:58441/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 68795 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 68802 T117 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 68809 T117 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 68826 T117 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 68831 T117 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 68842 T117 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 68864 T117 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 68867 T117 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57950_
   [junit4]   2> 68879 T117 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57950_
   [junit4]   2> 68883 T117 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 68892 T117 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 68902 T117 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:57950_
   [junit4]   2> 68902 T117 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 68905 T117 oasc.Overseer.start Overseer (id=91450090440425476-127.0.0.1:57950_-n_0000000000) starting
   [junit4]   2> 68915 T117 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 68945 T150 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 68946 T117 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 68948 T150 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 68951 T117 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 68954 T117 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 68962 T149 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 68974 T151 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 68974 T151 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 68977 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 68977 T151 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 68980 T149 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 68981 T149 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:57950",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:57950_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 68981 T149 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 68991 T149 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 68997 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 69002 T148 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 69978 T151 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 69978 T151 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1395417639993/collection1
   [junit4]   2> 69978 T151 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 69979 T151 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 69980 T151 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 69981 T151 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 69982 T151 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1395417639993/collection1/'
   [junit4]   2> 70037 T151 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 70107 T151 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 70110 T151 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 70188 T151 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 70260 T151 oasc.SolrResourceLoader.findClass WARN Solr loaded a deprecated plugin/analysis class [solr.SortableIntField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 70271 T151 oasc.SolrResourceLoader.findClass WARN Solr loaded a deprecated plugin/analysis class [solr.SortableLongField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 70282 T151 oasc.SolrResourceLoader.findClass WARN Solr loaded a deprecated plugin/analysis class [solr.SortableFloatField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 70293 T151 oasc.SolrResourceLoader.findClass WARN Solr loaded a deprecated plugin/analysis class [solr.SortableDoubleField]. Please consult documentation how to replace it accordingly.
   [junit4]   2> 70686 T151 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 70690 T151 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 70693 T151 oass.IndexSchema.readSchema WARN Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 70693 T151 oass.IndexSchema.readSchema WARN Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 70698 T151 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 70698 T151 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-controljetty-1395417639993/collection1/, dataDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/control/data/
   [junit4]   2> 70698 T151 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 70699 T151 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/control/data
   [junit4]   2> 70699 T151 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/control/data/index/
   [junit4]   2> 70699 T151 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 70700 T151 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/control/data/index
   [junit4]   2> 70701 T151 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@17b2be48 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/control/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 70701 T151 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 70703 T151 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 70703 T151 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 70703 T151 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 70704 T151 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 70704 T151 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 70704 T151 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 70705 T151 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 70705 T151 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: solr.PingRequestHandler
   [junit4]   2> 70707 T151 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70710 T151 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 70710 T151 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 70711 T151 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@17b2be48 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/control/data/index),segFN=segments_1,generation=1}
   [junit4]   2> 70711 T151 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 70711 T151 oass.SolrIndexSearcher.<init> Opening Searcher@61f460c1[collection1] main
   [junit4]   2> 70712 T152 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@61f460c1[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 70712 T151 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 70713 T117 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0
   [junit4]   2> 70713 T117 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 70713 T155 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:57950 collection:control_collection shard:shard1
   [junit4]   2> 70722 T155 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 70724 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 70727 T157 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@663281f5 name:ZooKeeperConnection Watcher:127.0.0.1:58441/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 70727 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 70729 T117 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 70738 T117 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 70739 T155 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 70743 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 70744 T155 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 70744 T155 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 70745 T149 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C10 name=collection1 org.apache.solr.core.SolrCore@70760c9f url=https://127.0.0.1:57950/collection1 node=127.0.0.1:57950_ C10_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:57950, core=collection1, node_name=127.0.0.1:57950_}
   [junit4]   2> 70747 T155 C10 P57950 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:57950/collection1/
   [junit4]   2> 70749 T155 C10 P57950 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 70750 T155 C10 P57950 oasc.SyncStrategy.syncToMe https://127.0.0.1:57950/collection1/ has no replicas
   [junit4]   2> 70751 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 70751 T155 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:57950/collection1/ shard1
   [junit4]   2> 70754 T155 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 70772 T117 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 70773 T117 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 70778 T117 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 70780 T117 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:45967
   [junit4]   2> 70784 T117 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 70784 T117 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 70785 T117 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1395417642121
   [junit4]   2> 70785 T117 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1395417642121/'
   [junit4]   2> 70797 T117 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1395417642121/solr.xml
   [junit4]   2> 70820 T157 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70820 T148 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70861 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 70863 T149 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 70871 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 70883 T117 oasc.CoreContainer.<init> New CoreContainer 1874907197
   [junit4]   2> 70884 T117 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1395417642121/]
   [junit4]   2> 70885 T117 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 70885 T117 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 70886 T117 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 70886 T117 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 70886 T117 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 70886 T117 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 70887 T117 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 70887 T117 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 70887 T117 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 70890 T117 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 70890 T117 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 70891 T117 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 70891 T117 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58441/solr
   [junit4]   2> 70891 T117 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 70892 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 70906 T168 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ffc30a1 name:ZooKeeperConnection Watcher:127.0.0.1:58441 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 70907 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 70910 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 70915 T170 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5e41f3fb name:ZooKeeperConnection Watcher:127.0.0.1:58441/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 70916 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 70927 T117 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 70974 T170 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70974 T157 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70974 T148 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 71013 T155 oasc.ZkController.register We are https://127.0.0.1:57950/collection1/ and leader is https://127.0.0.1:57950/collection1/
   [junit4]   2> 71013 T155 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:57950
   [junit4]   2> 71013 T155 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 71014 T155 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 71014 T155 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 71016 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71016 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71016 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71018 T155 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 71021 T149 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 71022 T149 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:57950",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:57950_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 71026 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71129 T148 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 71129 T157 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 71130 T170 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 71932 T117 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45967_
   [junit4]   2> 71934 T117 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45967_
   [junit4]   2> 71937 T157 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 71937 T148 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 71938 T170 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 71948 T171 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 71948 T171 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 71951 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71951 T171 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 71951 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71952 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71954 T149 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 71955 T149 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:45967",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45967_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 71956 T149 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 71956 T149 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 71960 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 72063 T170 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 72063 T157 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 72063 T148 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 72952 T171 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 72952 T171 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1395417642121/collection1
   [junit4]   2> 72952 T171 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 72953 T171 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 72953 T171 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 72955 T171 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 72955 T171 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1395417642121/collection1/'
   [junit4]   2> 72987 T171 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 73023 T171 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 73025 T171 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 73047 T171 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 73595 T171 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 73599 T171 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 73602 T171 oass.IndexSchema.readSchema WARN Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 73603 T171 oass.IndexSchema.readSchema WARN Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 73604 T171 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 73605 T171 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty1-1395417642121/collection1/, dataDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty1/
   [junit4]   2> 73606 T171 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 73606 T171 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty1
   [junit4]   2> 73607 T171 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty1/index/
   [junit4]   2> 73607 T171 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 73607 T171 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty1/index
   [junit4]   2> 73611 T171 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2b2e24aa lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty1/index),segFN=segments_1,generation=1}
   [junit4]   2> 73612 T171 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 73620 T171 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 73621 T171 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 73621 T171 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 73621 T171 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 73622 T171 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 73622 T171 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 73623 T171 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 73623 T171 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: solr.PingRequestHandler
   [junit4]   2> 73627 T171 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 73631 T171 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 73631 T171 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 73632 T171 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2b2e24aa lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty1/index),segFN=segments_1,generation=1}
   [junit4]   2> 73633 T171 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 73633 T171 oass.SolrIndexSearcher.<init> Opening Searcher@62f72d3[collection1] main
   [junit4]   2> 73634 T172 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@62f72d3[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 73634 T171 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 73635 T117 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0
   [junit4]   2> 73635 T117 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 73635 T175 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:45967 collection:collection1 shard:shard2
   [junit4]   2> 73638 T175 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 73650 T175 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 73653 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 73654 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 73654 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 73655 T175 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 73655 T175 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C11 name=collection1 org.apache.solr.core.SolrCore@38fc6bbb url=https://127.0.0.1:45967/collection1 node=127.0.0.1:45967_ C11_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:45967, core=collection1, node_name=127.0.0.1:45967_}
   [junit4]   2> 73656 T175 C11 P45967 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:45967/collection1/
   [junit4]   2> 73656 T149 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 73656 T175 C11 P45967 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 73657 T175 C11 P45967 oasc.SyncStrategy.syncToMe https://127.0.0.1:45967/collection1/ has no replicas
   [junit4]   2> 73657 T175 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:45967/collection1/ shard2
   [junit4]   2> 73658 T175 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 73667 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 73670 T170 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 73670 T148 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 73670 T157 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 73678 T149 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 73683 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 73688 T117 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 73691 T117 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 73695 T117 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 73698 T117 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:50531
   [junit4]   2> 73700 T117 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 73701 T117 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 73701 T117 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1395417645019
   [junit4]   2> 73702 T117 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1395417645019/'
   [junit4]   2> 73715 T117 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1395417645019/solr.xml
   [junit4]   2> 73756 T117 oasc.CoreContainer.<init> New CoreContainer 1176142212
   [junit4]   2> 73756 T117 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1395417645019/]
   [junit4]   2> 73757 T117 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 73757 T117 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 73758 T117 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 73758 T117 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 73758 T117 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 73758 T117 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 73759 T117 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 73759 T117 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 73759 T117 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 73762 T117 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 73762 T117 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 73763 T117 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 73763 T117 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58441/solr
   [junit4]   2> 73763 T117 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 73764 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 73766 T186 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@ba8707d name:ZooKeeperConnection Watcher:127.0.0.1:58441 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 73766 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 73769 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 73771 T188 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b038c21 name:ZooKeeperConnection Watcher:127.0.0.1:58441/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 73771 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 73781 T117 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 73786 T170 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 73787 T157 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 73786 T188 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 73786 T148 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 73824 T175 oasc.ZkController.register We are https://127.0.0.1:45967/collection1/ and leader is https://127.0.0.1:45967/collection1/
   [junit4]   2> 73825 T175 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:45967
   [junit4]   2> 73825 T175 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 73825 T175 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 73826 T175 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 73827 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 73828 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 73828 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 73828 T175 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 73834 T149 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 73836 T149 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:45967",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45967_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 73840 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 73945 T170 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 73946 T148 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 73945 T157 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 73946 T188 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 74785 T117 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50531_
   [junit4]   2> 74787 T117 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50531_
   [junit4]   2> 74791 T148 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 74791 T157 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 74791 T170 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 74791 T188 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 74803 T189 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 74804 T189 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 74805 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 74805 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 74805 T189 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 74805 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 74808 T149 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 74809 T149 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:50531",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50531_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 74809 T149 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 74809 T149 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 74812 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 74916 T188 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 74916 T157 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 74916 T148 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 74916 T170 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 75806 T189 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 75806 T189 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1395417645019/collection1
   [junit4]   2> 75806 T189 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 75807 T189 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 75808 T189 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 75809 T189 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 75809 T189 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1395417645019/collection1/'
   [junit4]   2> 75852 T189 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 75889 T189 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 75891 T189 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 75916 T189 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 76271 T189 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 76273 T189 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 76275 T189 oass.IndexSchema.readSchema WARN Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 76275 T189 oass.IndexSchema.readSchema WARN Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 76276 T189 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 76277 T189 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty2-1395417645019/collection1/, dataDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty2/
   [junit4]   2> 76277 T189 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 76277 T189 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty2
   [junit4]   2> 76278 T189 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty2/index/
   [junit4]   2> 76278 T189 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 76278 T189 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty2/index
   [junit4]   2> 76279 T189 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@525534db lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty2/index),segFN=segments_1,generation=1}
   [junit4]   2> 76280 T189 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 76281 T189 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 76281 T189 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 76281 T189 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 76281 T189 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 76282 T189 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 76282 T189 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 76282 T189 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 76283 T189 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: solr.PingRequestHandler
   [junit4]   2> 76285 T189 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 76288 T189 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 76288 T189 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 76289 T189 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@525534db lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty2/index),segFN=segments_1,generation=1}
   [junit4]   2> 76289 T189 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 76289 T189 oass.SolrIndexSearcher.<init> Opening Searcher@7f02a3da[collection1] main
   [junit4]   2> 76290 T190 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7f02a3da[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 76290 T189 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 76291 T117 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0
   [junit4]   2> 76292 T117 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 76292 T193 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:50531 collection:collection1 shard:shard1
   [junit4]   2> 76294 T193 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 76302 T193 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 76306 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 76307 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 76307 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 76308 T193 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 76308 T193 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C12 name=collection1 org.apache.solr.core.SolrCore@53fafc0d url=https://127.0.0.1:50531/collection1 node=127.0.0.1:50531_ C12_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:50531, core=collection1, node_name=127.0.0.1:50531_}
   [junit4]   2> 76308 T193 C12 P50531 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:50531/collection1/
   [junit4]   2> 76309 T193 C12 P50531 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 76309 T193 C12 P50531 oasc.SyncStrategy.syncToMe https://127.0.0.1:50531/collection1/ has no replicas
   [junit4]   2> 76309 T193 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:50531/collection1/ shard1
   [junit4]   2> 76309 T149 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 76310 T193 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 76319 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 76322 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 76327 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 76343 T117 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 76344 T117 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 76354 T117 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 76358 T117 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:47115
   [junit4]   2> 76363 T117 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 76363 T117 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 76364 T117 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1395417647676
   [junit4]   2> 76364 T117 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1395417647676/'
   [junit4]   2> 76378 T117 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0/./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1395417647676/solr.xml
   [junit4]   2> 76432 T157 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 76432 T170 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 76432 T148 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 76432 T188 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 76445 T117 oasc.CoreContainer.<init> New CoreContainer 386075102
   [junit4]   2> 76446 T117 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1395417647676/]
   [junit4]   2> 76448 T117 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 76448 T117 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 76449 T117 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 76449 T117 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 76449 T117 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 76450 T117 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 76450 T117 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 76451 T117 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 76451 T117 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 76455 T117 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 76456 T117 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 76456 T117 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 76457 T117 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58441/solr
   [junit4]   2> 76457 T117 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 76460 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 76461 T204 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@43594221 name:ZooKeeperConnection Watcher:127.0.0.1:58441 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 76462 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 76466 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 76467 T206 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15ecea8a name:ZooKeeperConnection Watcher:127.0.0.1:58441/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 76467 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 76474 T193 oasc.ZkController.register We are https://127.0.0.1:50531/collection1/ and leader is https://127.0.0.1:50531/collection1/
   [junit4]   2> 76474 T193 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:50531
   [junit4]   2> 76475 T193 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 76475 T193 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 76475 T193 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 76478 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 76479 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 76479 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 76481 T193 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 76482 T149 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 76483 T149 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:50531",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50531_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 76489 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 76493 T117 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 76592 T170 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 76593 T157 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 76592 T148 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 76592 T188 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 76593 T206 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 77507 T117 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47115_
   [junit4]   2> 77509 T117 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:47115_
   [junit4]   2> 77512 T170 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 77512 T188 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 77512 T157 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 77512 T206 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 77513 T148 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 77522 T207 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 77522 T207 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 77524 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77525 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77525 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77525 T207 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 77527 T149 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 77528 T149 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:47115",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:47115_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 77528 T149 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 77528 T149 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 77531 T148 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 77633 T206 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 77634 T188 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 77634 T170 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 77634 T157 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 77633 T148 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 78526 T207 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 78526 T207 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1395417647676/collection1
   [junit4]   2> 78526 T207 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 78527 T207 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 78527 T207 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 78529 T207 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 78529 T207 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1395417647676/collection1/'
   [junit4]   2> 78567 T207 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 78616 T207 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 78618 T207 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 78638 T207 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 79092 T207 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 79094 T207 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 79096 T207 oass.IndexSchema.readSchema WARN Field lowerfilt1and2 is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 79097 T207 oass.IndexSchema.readSchema WARN Field text is not multivalued and destination for multiple copyFields (3)
   [junit4]   2> 79098 T207 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 79098 T207 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-jetty3-1395417647676/collection1/, dataDir=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty3/
   [junit4]   2> 79098 T207 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 79098 T207 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty3
   [junit4]   2> 79099 T207 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty3/index/
   [junit4]   2> 79099 T207 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty3/index' doesn't exist. Creating new index...
   [junit4]   2> 79099 T207 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty3/index
   [junit4]   2> 79100 T207 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2d153029 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty3/index),segFN=segments_1,generation=1}
   [junit4]   2> 79101 T207 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 79102 T207 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 79102 T207 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 79102 T207 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 79102 T207 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 79103 T207 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 79103 T207 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 79103 T207 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 79103 T207 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: solr.PingRequestHandler
   [junit4]   2> 79106 T207 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 79108 T207 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 79108 T207 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 79109 T207 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2d153029 lockFactory=NativeFSLockFactory@./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty3/index),segFN=segments_1,generation=1}
   [junit4]   2> 79109 T207 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 79110 T207 oass.SolrIndexSearcher.<init> Opening Searcher@5770eecb[collection1] main
   [junit4]   2> 79111 T208 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5770eecb[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 79111 T207 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 79112 T117 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-solrj/test/J0
   [junit4]   2> 79112 T117 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 79112 T211 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:47115 collection:collection1 shard:shard2
   [junit4]   2> 79117 T211 oasc.ZkController.register We are https://127.0.0.1:47115/col

[...truncated too long message...]

ction1, node_name=127.0.0.1:47115_}
   [junit4]   2> 97407 T206 C30 P47115 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
   [junit4]   2> 97411 T206 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is shutdown
   [junit4]   2> 97412 T206 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@367cc540
   [junit4]   2> 97412 T206 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=6,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=2,cumulative_deletesById=1,cumulative_deletesByQuery=2,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=5}
   [junit4]   2> 97412 T206 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 97413 T206 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 97413 T206 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 97414 T206 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 97414 T206 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 97414 T206 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty3/index [CachedDir<<refCount=0;path=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty3/index;done=false>>]
   [junit4]   2> 97415 T206 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty3/index
   [junit4]   2> 97415 T206 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty3 [CachedDir<<refCount=0;path=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty3;done=false>>]
   [junit4]   2> 97415 T206 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty3
   [junit4]   2> 97415 T206 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 97415 T206 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 97416 T206 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 97416 T206 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:259)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> ASYNC  NEW_CORE C31 name=collection1 org.apache.solr.core.SolrCore@199eb207 url=https://127.0.0.1:54515/collection1 node=127.0.0.1:54515_ C31_STATE=coll:collection1 core:collection1 props:{state=active, base_url=https://127.0.0.1:54515, core=collection1, node_name=127.0.0.1:54515_}
   [junit4]   2> 97492 T225 C31 P54515 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
   [junit4]   2> 97493 T225 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is shutdown
   [junit4]   2> 97494 T225 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@199eb207
   [junit4]   2> 97495 T225 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=6,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=2,cumulative_deletesById=1,cumulative_deletesByQuery=2,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=5}
   [junit4]   2> 97495 T225 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 97495 T225 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 97496 T225 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 97497 T225 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 97498 T225 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 97498 T225 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty4/index [CachedDir<<refCount=0;path=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty4/index;done=false>>]
   [junit4]   2> 97498 T225 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty4/index
   [junit4]   2> 97498 T225 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty4 [CachedDir<<refCount=0;path=./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty4;done=false>>]
   [junit4]   2> 97499 T225 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.client.solrj.impl.CloudSolrServerTest-1395417639775/jetty4
   [junit4]   2> 97499 T225 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:259)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 97500 T225 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: test params are: codec=Lucene40, sim=DefaultSimilarity, locale=es_PE, timezone=America/New_York
   [junit4]   2> NOTE: Linux 3.8.0-37-generic amd64/Oracle Corporation 1.7.0_51 (64-bit)/cpus=8,threads=1,free=91819736,total=131072000
   [junit4]   2> NOTE: All tests run in this JVM: [SolrParamTest, ClientUtilsTest, TestDocumentObjectBinder, TestEmbeddedSolrServer, LargeVolumeBinaryJettyTest, AnlysisResponseBaseTest, LargeVolumeEmbeddedTest, ModifiableSolrParamsTest, TestXMLEscaping, DocumentAnalysisResponseTest, SolrExampleXMLTest, TestCoreAdmin, CloudSolrServerTest]
   [junit4] Completed on J0 in 77.81s, 2 tests, 1 error <<< FAILURES!

[...truncated 97 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:202: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:490: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1276: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:908: There were test failures: 49 suites, 283 tests, 1 error

Total time: 53 minutes 37 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0_51 -XX:+UseCompressedOops -XX:+UseG1GC -XX:-UseSuperWord
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure