You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2013/07/21 22:53:27 UTC
[JENKINS] Lucene-Solr-Tests-trunk-Java7 - Build # 4163 - Failure
Build: https://builds.apache.org/job/Lucene-Solr-Tests-trunk-Java7/4163/
2 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.BasicDistributedZkTest
Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest: 1) Thread[id=2759, name=recoveryCmdExecutor-1327-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at java.net.PlainSocketImpl.socketConnect(Native Method) at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339) at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200) at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182) at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391) at java.net.Socket.connect(Socket.java:579) at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127) at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180) at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294) at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645) at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180) at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:722)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest:
1) Thread[id=2759, name=recoveryCmdExecutor-1327-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391)
at java.net.Socket.connect(Socket.java:579)
at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:722)
at __randomizedtesting.SeedInfo.seed([CE9EC1F579D24215]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.BasicDistributedZkTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=2759, name=recoveryCmdExecutor-1327-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at java.net.PlainSocketImpl.socketConnect(Native Method) at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339) at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200) at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182) at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391) at java.net.Socket.connect(Socket.java:579) at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127) at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180) at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294) at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645) at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180) at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:722)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=2759, name=recoveryCmdExecutor-1327-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391)
at java.net.Socket.connect(Socket.java:579)
at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:722)
at __randomizedtesting.SeedInfo.seed([CE9EC1F579D24215]:0)
Build Log:
[...truncated 9651 lines...]
[junit4] Suite: org.apache.solr.cloud.BasicDistributedZkTest
[junit4] 2> 1239091 T2315 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /i_u/c
[junit4] 2> 1239095 T2315 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4] 2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZkTest-1374439712923
[junit4] 2> 1239097 T2315 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 1239097 T2316 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 1239198 T2315 oasc.ZkTestServer.run start zk server on port:33125
[junit4] 2> 1239199 T2315 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1239228 T2322 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f174eb4 name:ZooKeeperConnection Watcher:127.0.0.1:33125 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1239229 T2315 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1239229 T2315 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 1239239 T2315 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1239241 T2324 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2905d969 name:ZooKeeperConnection Watcher:127.0.0.1:33125/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1239241 T2315 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1239241 T2315 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4] 2> 1239245 T2315 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4] 2> 1239247 T2315 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4] 2> 1239255 T2315 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4] 2> 1239258 T2315 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1239258 T2315 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4] 2> 1239271 T2315 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 1239272 T2315 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4] 2> 1239401 T2315 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1239403 T2315 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4] 2> 1239406 T2315 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1239407 T2315 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4] 2> 1239410 T2315 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1239411 T2315 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4] 2> 1239414 T2315 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1239414 T2315 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4] 2> 1239417 T2315 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1239418 T2315 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1239421 T2315 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1239422 T2315 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4] 2> 1239430 T2315 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1239431 T2315 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4] 2> 1239717 T2315 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 1239724 T2315 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:16941
[junit4] 2> 1239725 T2315 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 1239725 T2315 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 1239726 T2315 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374439713263
[junit4] 2> 1239727 T2315 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374439713263/'
[junit4] 2> 1239765 T2315 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374439713263/solr.xml
[junit4] 2> 1239857 T2315 oasc.CoreContainer.<init> New CoreContainer 1627881979
[junit4] 2> 1239857 T2315 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374439713263/]
[junit4] 2> 1239859 T2315 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4] 2> 1239860 T2315 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 1239861 T2315 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 1239861 T2315 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 1239862 T2315 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 1239862 T2315 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 1239863 T2315 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 1239863 T2315 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 1239864 T2315 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 1239864 T2315 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4] 2> 1239880 T2315 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 1239881 T2315 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33125/solr
[junit4] 2> 1239881 T2315 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 1239883 T2315 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1239885 T2335 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e6faf02 name:ZooKeeperConnection Watcher:127.0.0.1:33125 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1239885 T2315 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1239888 T2315 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4] 2> 1239895 T2315 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1239897 T2337 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@20e0b471 name:ZooKeeperConnection Watcher:127.0.0.1:33125/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1239897 T2315 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1239900 T2315 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 1239909 T2315 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 1239913 T2315 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 1239920 T2315 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:16941_i_u%2Fc
[junit4] 2> 1239921 T2315 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:16941_i_u%2Fc
[junit4] 2> 1239925 T2315 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 1239944 T2315 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 1239947 T2315 oasc.Overseer.start Overseer (id=90075281026580483-127.0.0.1:16941_i_u%2Fc-n_0000000000) starting
[junit4] 2> 1239959 T2315 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 1239968 T2339 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 1239969 T2315 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 1239972 T2315 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 1239974 T2315 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1239976 T2338 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 1239982 T2340 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 1239982 T2340 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 1239984 T2340 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 1241480 T2338 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1241482 T2338 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "numShards":"1",
[junit4] 2> "shard":null,
[junit4] 2> "roles":null,
[junit4] 2> "state":"down",
[junit4] 2> "shard_state":"active",
[junit4] 2> "core":"collection1",
[junit4] 2> "collection":"control_collection",
[junit4] 2> "shard_range":null,
[junit4] 2> "node_name":"127.0.0.1:16941_i_u%2Fc",
[junit4] 2> "base_url":"http://127.0.0.1:16941/i_u/c"}
[junit4] 2> 1241482 T2338 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
[junit4] 2> 1241482 T2338 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 1241499 T2337 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1241986 T2340 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
[junit4] 2> 1241986 T2340 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374439713263/collection1
[junit4] 2> 1241986 T2340 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4] 2> 1241987 T2340 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4] 2> 1241988 T2340 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4] 2> 1241989 T2340 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374439713263/collection1/'
[junit4] 2> 1241991 T2340 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374439713263/collection1/lib/README' to classloader
[junit4] 2> 1241991 T2340 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374439713263/collection1/lib/classes/' to classloader
[junit4] 2> 1242054 T2340 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4] 2> 1242124 T2340 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4] 2> 1242225 T2340 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4] 2> 1242239 T2340 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4] 2> 1242901 T2340 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4] 2> 1242916 T2340 oass.IndexSchema.readSchema default search field in schema is text
[junit4] 2> 1242920 T2340 oass.IndexSchema.readSchema unique key field: id
[junit4] 2> 1242937 T2340 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 1242942 T2340 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 1242946 T2340 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 1242947 T2340 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 1242947 T2340 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 1242948 T2340 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 1242948 T2340 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 1242949 T2340 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 1242949 T2340 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4] 2> 1242949 T2340 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374439713263/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/control/data/
[junit4] 2> 1242950 T2340 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@71008045
[junit4] 2> 1242951 T2340 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/control/data
[junit4] 2> 1242951 T2340 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/control/data/index/
[junit4] 2> 1242951 T2340 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/control/data/index' doesn't exist. Creating new index...
[junit4] 2> 1242952 T2340 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/control/data/index
[junit4] 2> 1242956 T2340 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@52d2fa63 lockFactory=org.apache.lucene.store.NativeFSLockFactory@506ad72f),segFN=segments_1,generation=1}
[junit4] 2> 1242956 T2340 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 1242961 T2340 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4] 2> 1242962 T2340 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4] 2> 1242962 T2340 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 1242963 T2340 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4] 2> 1242964 T2340 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4] 2> 1242964 T2340 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4] 2> 1242964 T2340 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4] 2> 1242965 T2340 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4] 2> 1242965 T2340 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4] 2> 1242980 T2340 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4] 2> 1242987 T2340 oass.SolrIndexSearcher.<init> Opening Searcher@5713a4a2 main
[junit4] 2> 1242988 T2340 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4] 2> 1242989 T2340 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4] 2> 1242992 T2341 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5713a4a2 main{StandardDirectoryReader(segments_1:1)}
[junit4] 2> 1242994 T2340 oasc.CoreContainer.registerCore registering core: collection1
[junit4] 2> 1242994 T2340 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:16941/i_u/c collection:control_collection shard:shard1
[junit4] 2> 1242994 T2340 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
[junit4] 2> 1243011 T2340 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4] 2> 1243032 T2340 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
[junit4] 2> 1243034 T2340 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4] 2> 1243035 T2340 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4] 2> 1243035 T2340 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:16941/i_u/c/collection1/
[junit4] 2> 1243035 T2340 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4] 2> 1243035 T2340 oasc.SyncStrategy.syncToMe http://127.0.0.1:16941/i_u/c/collection1/ has no replicas
[junit4] 2> 1243035 T2340 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:16941/i_u/c/collection1/ shard1
[junit4] 2> 1243036 T2340 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4] 2> 1244505 T2338 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1244535 T2337 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1244572 T2340 oasc.ZkController.register We are http://127.0.0.1:16941/i_u/c/collection1/ and leader is http://127.0.0.1:16941/i_u/c/collection1/
[junit4] 2> 1244572 T2340 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:16941/i_u/c
[junit4] 2> 1244572 T2340 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4] 2> 1244572 T2340 oasc.ZkController.publish publishing core=collection1 state=active
[junit4] 2> 1244573 T2340 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 1244575 T2340 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1244576 T2315 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0
[junit4] 2> 1244577 T2315 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4] 2> 1244577 T2315 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4] 2> 1244590 T2315 oascsi.HttpClientUtil.createClient Creating new http client, config:
[junit4] 2> 1244591 T2315 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1244594 T2344 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@48ab5a2b name:ZooKeeperConnection Watcher:127.0.0.1:33125/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1244594 T2315 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1244596 T2315 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1244598 T2315 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
[junit4] 2> 1244884 T2315 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 1244887 T2315 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:52128
[junit4] 2> 1244888 T2315 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 1244888 T2315 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 1244889 T2315 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374439718426
[junit4] 2> 1244889 T2315 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374439718426/'
[junit4] 2> 1244927 T2315 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374439718426/solr.xml
[junit4] 2> 1245032 T2315 oasc.CoreContainer.<init> New CoreContainer 732526369
[junit4] 2> 1245032 T2315 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374439718426/]
[junit4] 2> 1245034 T2315 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4] 2> 1245035 T2315 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 1245035 T2315 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 1245036 T2315 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 1245036 T2315 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 1245037 T2315 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 1245037 T2315 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 1245038 T2315 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 1245038 T2315 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 1245039 T2315 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4] 2> 1245054 T2315 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 1245055 T2315 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33125/solr
[junit4] 2> 1245056 T2315 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 1245057 T2315 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1245059 T2355 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@844e790 name:ZooKeeperConnection Watcher:127.0.0.1:33125 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1245060 T2315 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1245062 T2315 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4] 2> 1245073 T2315 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1245075 T2357 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@333230dc name:ZooKeeperConnection Watcher:127.0.0.1:33125/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1245075 T2315 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1245087 T2315 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1246040 T2338 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1246042 T2338 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "numShards":"1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "roles":null,
[junit4] 2> "state":"active",
[junit4] 2> "shard_state":"active",
[junit4] 2> "core":"collection1",
[junit4] 2> "collection":"control_collection",
[junit4] 2> "shard_range":null,
[junit4] 2> "node_name":"127.0.0.1:16941_i_u%2Fc",
[junit4] 2> "base_url":"http://127.0.0.1:16941/i_u/c"}
[junit4] 2> 1246056 T2337 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1246056 T2344 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1246056 T2357 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1246091 T2315 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52128_i_u%2Fc
[junit4] 2> 1246093 T2315 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:52128_i_u%2Fc
[junit4] 2> 1246095 T2357 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1246096 T2337 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 1246096 T2344 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 1246096 T2337 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4] 2> 1246096 T2344 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4] 2> 1246097 T2357 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 1246111 T2358 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 1246112 T2358 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 1246113 T2358 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 1247561 T2338 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1247563 T2338 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "numShards":"2",
[junit4] 2> "shard":null,
[junit4] 2> "roles":null,
[junit4] 2> "state":"down",
[junit4] 2> "shard_state":"active",
[junit4] 2> "core":"collection1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard_range":null,
[junit4] 2> "node_name":"127.0.0.1:52128_i_u%2Fc",
[junit4] 2> "base_url":"http://127.0.0.1:52128/i_u/c"}
[junit4] 2> 1247563 T2338 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 2
[junit4] 2> 1247563 T2338 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 1247584 T2337 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 1247584 T2357 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 1247584 T2344 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 1248115 T2358 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
[junit4] 2> 1248115 T2358 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374439718426/collection1
[junit4] 2> 1248115 T2358 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4] 2> 1248116 T2358 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4] 2> 1248117 T2358 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4] 2> 1248118 T2358 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374439718426/collection1/'
[junit4] 2> 1248119 T2358 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374439718426/collection1/lib/README' to classloader
[junit4] 2> 1248120 T2358 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374439718426/collection1/lib/classes/' to classloader
[junit4] 2> 1248183 T2358 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4] 2> 1248253 T2358 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4] 2> 1248355 T2358 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4] 2> 1248369 T2358 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4] 2> 1248996 T2358 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4] 2> 1249006 T2358 oass.IndexSchema.readSchema default search field in schema is text
[junit4] 2> 1249008 T2358 oass.IndexSchema.readSchema unique key field: id
[junit4] 2> 1249022 T2358 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 1249025 T2358 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 1249029 T2358 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 1249030 T2358 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 1249030 T2358 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 1249030 T2358 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 1249031 T2358 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 1249031 T2358 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 1249032 T2358 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4] 2> 1249032 T2358 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374439718426/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/jetty1/
[junit4] 2> 1249032 T2358 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@71008045
[junit4] 2> 1249033 T2358 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/jetty1
[junit4] 2> 1249033 T2358 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/jetty1/index/
[junit4] 2> 1249033 T2358 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/jetty1/index' doesn't exist. Creating new index...
[junit4] 2> 1249034 T2358 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/jetty1/index
[junit4] 2> 1249036 T2358 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@4fb0374a lockFactory=org.apache.lucene.store.NativeFSLockFactory@73a118bc),segFN=segments_1,generation=1}
[junit4] 2> 1249037 T2358 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 1249041 T2358 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4] 2> 1249041 T2358 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4] 2> 1249042 T2358 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 1249042 T2358 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4] 2> 1249043 T2358 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4] 2> 1249043 T2358 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4] 2> 1249043 T2358 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4] 2> 1249043 T2358 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4] 2> 1249044 T2358 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4] 2> 1249054 T2358 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4] 2> 1249059 T2358 oass.SolrIndexSearcher.<init> Opening Searcher@65f5d6e0 main
[junit4] 2> 1249060 T2358 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4] 2> 1249060 T2358 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4] 2> 1249064 T2359 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@65f5d6e0 main{StandardDirectoryReader(segments_1:1)}
[junit4] 2> 1249066 T2358 oasc.CoreContainer.registerCore registering core: collection1
[junit4] 2> 1249066 T2358 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:52128/i_u/c collection:collection1 shard:shard1
[junit4] 2> 1249066 T2358 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
[junit4] 2> 1249078 T2358 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 1249114 T2358 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
[junit4] 2> 1249125 T2358 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4] 2> 1249126 T2358 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4] 2> 1249126 T2358 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:52128/i_u/c/collection1/
[junit4] 2> 1249126 T2358 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4] 2> 1249126 T2358 oasc.SyncStrategy.syncToMe http://127.0.0.1:52128/i_u/c/collection1/ has no replicas
[junit4] 2> 1249127 T2358 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:52128/i_u/c/collection1/ shard1
[junit4] 2> 1249127 T2358 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1250603 T2338 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1250619 T2337 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 1250619 T2344 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 1250619 T2357 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 1250663 T2358 oasc.ZkController.register We are http://127.0.0.1:52128/i_u/c/collection1/ and leader is http://127.0.0.1:52128/i_u/c/collection1/
[junit4] 2> 1250663 T2358 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:52128/i_u/c
[junit4] 2> 1250663 T2358 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4] 2> 1250663 T2358 oasc.ZkController.publish publishing core=collection1 state=active
[junit4] 2> 1250664 T2358 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 1250666 T2358 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1250667 T2315 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0
[junit4] 2> 1250667 T2315 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4] 2> 1250668 T2315 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4] 2> 1250954 T2315 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 1250957 T2315 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:58870
[junit4] 2> 1250958 T2315 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 1250958 T2315 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 1250959 T2315 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374439724507
[junit4] 2> 1250959 T2315 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374439724507/'
[junit4] 2> 1250997 T2315 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374439724507/solr.xml
[junit4] 2> 1251088 T2315 oasc.CoreContainer.<init> New CoreContainer 186115700
[junit4] 2> 1251089 T2315 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374439724507/]
[junit4] 2> 1251091 T2315 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4] 2> 1251091 T2315 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 1251092 T2315 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 1251092 T2315 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 1251093 T2315 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 1251093 T2315 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 1251094 T2315 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 1251094 T2315 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 1251095 T2315 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 1251095 T2315 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4] 2> 1251113 T2315 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 1251113 T2315 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33125/solr
[junit4] 2> 1251114 T2315 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 1251115 T2315 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1251118 T2371 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b662051 name:ZooKeeperConnection Watcher:127.0.0.1:33125 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1251118 T2315 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1251130 T2315 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4] 2> 1251141 T2315 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1251144 T2373 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5092067d name:ZooKeeperConnection Watcher:127.0.0.1:33125/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1251144 T2315 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1251150 T2315 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1252131 T2338 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1252133 T2338 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "numShards":"2",
[junit4] 2> "shard":"shard1",
[junit4] 2> "roles":null,
[junit4] 2> "state":"active",
[junit4] 2> "shard_state":"active",
[junit4] 2> "core":"collection1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard_range":null,
[junit4] 2> "node_name":"127.0.0.1:52128_i_u%2Fc",
[junit4] 2> "base_url":"http://127.0.0.1:52128/i_u/c"}
[junit4] 2> 1252169 T2315 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58870_i_u%2Fc
[junit4] 2> 1252171 T2357 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 1252171 T2344 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 1252171 T2373 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 1252171 T2337 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 1252249 T2315 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:58870_i_u%2Fc
[junit4] 2> 1252277 T2357 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4] 2> 1252278 T2344 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 1252278 T2373 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 1252278 T2344 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1252278 T2337 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 1252279 T2373 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1252279 T2337 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1252279 T2357 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 1252287 T2374 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 1252288 T2374 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 1252290 T2374 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 1253753 T2338 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1253755 T2338 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "core_node_name":"core_node2",
[junit4] 2> "numShards":"2",
[junit4] 2> "shard":null,
[junit4] 2> "roles":null,
[junit4] 2> "state":"down",
[junit4] 2> "shard_state":"active",
[junit4] 2> "core":"collection1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard_range":null,
[junit4] 2> "node_name":"127.0.0.1:58870_i_u%2Fc",
[junit4] 2> "base_url":"http://127.0.0.1:58870/i_u/c"}
[junit4] 2> 1253755 T2338 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4] 2> 1253755 T2338 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
[junit4] 2> 1253781 T2357 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1253781 T2344 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1253781 T2337 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1253781 T2373 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1254292 T2374 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
[junit4] 2> 1254292 T2374 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374439724507/collection1
[junit4] 2> 1254292 T2374 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4] 2> 1254293 T2374 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4] 2> 1254294 T2374 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4] 2> 1254295 T2374 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374439724507/collection1/'
[junit4] 2> 1254296 T2374 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374439724507/collection1/lib/README' to classloader
[junit4] 2> 1254297 T2374 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374439724507/collection1/lib/classes/' to classloader
[junit4] 2> 1254354 T2374 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4] 2> 1254433 T2374 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4] 2> 1254534 T2374 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4] 2> 1254547 T2374 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4] 2> 1255173 T2374 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4] 2> 1255187 T2374 oass.IndexSchema.readSchema default search field in schema is text
[junit4] 2> 1255191 T2374 oass.IndexSchema.readSchema unique key field: id
[junit4] 2> 1255207 T2374 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 1255212 T2374 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 1255217 T2374 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 1255218 T2374 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 1255218 T2374 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 1255218 T2374 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 1255219 T2374 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 1255220 T2374 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 1255220 T2374 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4] 2> 1255220 T2374 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374439724507/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/jetty2/
[junit4] 2> 1255221 T2374 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@71008045
[junit4] 2> 1255222 T2374 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/jetty2
[junit4] 2> 1255222 T2374 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/jetty2/index/
[junit4] 2> 1255222 T2374 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/jetty2/index' doesn't exist. Creating new index...
[junit4] 2> 1255223 T2374 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/jetty2/index
[junit4] 2> 1255227 T2374 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@7d478a7f lockFactory=org.apache.lucene.store.NativeFSLockFactory@422426fb),segFN=segments_1,generation=1}
[junit4] 2> 1255227 T2374 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 1255233 T2374 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4] 2> 1255233 T2374 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4] 2> 1255234 T2374 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 1255235 T2374 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4] 2> 1255236 T2374 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4] 2> 1255236 T2374 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4] 2> 1255236 T2374 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4] 2> 1255237 T2374 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4] 2> 1255238 T2374 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4] 2> 1255254 T2374 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4] 2> 1255261 T2374 oass.SolrIndexSearcher.<init> Opening Searcher@54c96476 main
[junit4] 2> 1255263 T2374 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4] 2> 1255263 T2374 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4] 2> 1255268 T2375 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@54c96476 main{StandardDirectoryReader(segments_1:1)}
[junit4] 2> 1255270 T2374 oasc.CoreContainer.registerCore registering core: collection1
[junit4] 2> 1255271 T2374 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:58870/i_u/c collection:collection1 shard:shard2
[junit4] 2> 1255271 T2374 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
[junit4] 2> 1255289 T2374 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4] 2> 1255302 T2374 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
[junit4] 2> 1255310 T2374 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4] 2> 1255310 T2374 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4] 2> 1255311 T2374 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:58870/i_u/c/collection1/
[junit4] 2> 1255311 T2374 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4] 2> 1255311 T2374 oasc.SyncStrategy.syncToMe http://127.0.0.1:58870/i_u/c/collection1/ has no replicas
[junit4] 2> 1255311 T2374 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:58870/i_u/c/collection1/ shard2
[junit4] 2> 1255312 T2374 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4] 2> 1256793 T2338 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1256804 T2337 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1256804 T2344 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1256804 T2373 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1256804 T2357 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1256855 T2374 oasc.ZkController.register We are http://127.0.0.1:58870/i_u/c/collection1/ and leader is http://127.0.0.1:58870/i_u/c/collection1/
[junit4] 2> 1256855 T2374 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:58870/i_u/c
[junit4] 2> 1256855 T2374 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4] 2> 1256856 T2374 oasc.ZkController.publish publishing core=collection1 state=active
[junit4] 2> 1256856 T2374 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 1256858 T2374 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1256859 T2315 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0
[junit4] 2> 1256860 T2315 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4] 2> 1256860 T2315 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4] 2> 1257155 T2315 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 1257157 T2315 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:65079
[junit4] 2> 1257158 T2315 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 1257159 T2315 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 1257159 T2315 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374439730697
[junit4] 2> 1257160 T2315 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374439730697/'
[junit4] 2> 1257198 T2315 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374439730697/solr.xml
[junit4] 2> 1257288 T2315 oasc.CoreContainer.<init> New CoreContainer 131368618
[junit4] 2> 1257289 T2315 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374439730697/]
[junit4] 2> 1257291 T2315 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4] 2> 1257291 T2315 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 1257292 T2315 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 1257292 T2315 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 1257293 T2315 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 1257293 T2315 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 1257294 T2315 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 1257294 T2315 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 1257295 T2315 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 1257295 T2315 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4] 2> 1257311 T2315 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 1257311 T2315 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33125/solr
[junit4] 2> 1257312 T2315 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 1257313 T2315 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1257321 T2387 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7710ff84 name:ZooKeeperConnection Watcher:127.0.0.1:33125 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1257321 T2315 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1257337 T2315 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4] 2> 1257348 T2315 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1257350 T2389 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e501b2b name:ZooKeeperConnection Watcher:127.0.0.1:33125/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1257350 T2315 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1257362 T2315 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1258309 T2338 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1258311 T2338 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "core_node_name":"core_node2",
[junit4] 2> "numShards":"2",
[junit4] 2> "shard":"shard2",
[junit4] 2> "roles":null,
[junit4] 2> "state":"active",
[junit4] 2> "shard_state":"active",
[junit4] 2> "core":"collection1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard_range":null,
[junit4] 2> "node_name":"127.0.0.1:58870_i_u%2Fc",
[junit4] 2> "base_url":"http://127.0.0.1:58870/i_u/c"}
[junit4] 2> 1258342 T2389 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1258342 T2344 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1258342 T2373 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1258342 T2337 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1258342 T2357 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1258365 T2315 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65079_i_u%2Fc
[junit4] 2> 1258367 T2315 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:65079_i_u%2Fc
[junit4] 2> 1258374 T2357 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4] 2> 1258374 T2344 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 1258375 T2389 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 1258375 T2337 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 1258375 T2389 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4] 2> 1258375 T2344 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4] 2> 1258375 T2373 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 1258375 T2337 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4] 2> 1258376 T2373 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4] 2> 1258376 T2357 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 1258385 T2390 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 1258385 T2390 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 1258387 T2390 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 1259852 T2338 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1259854 T2338 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "core_node_name":"core_node3",
[junit4] 2> "numShards":"2",
[junit4] 2> "shard":null,
[junit4] 2> "roles":null,
[junit4] 2> "state":"down",
[junit4] 2> "shard_state":"active",
[junit4] 2> "core":"collection1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard_range":null,
[junit4] 2> "node_name":"127.0.0.1:65079_i_u%2Fc",
[junit4] 2> "base_url":"http://127.0.0.1:65079/i_u/c"}
[junit4] 2> 1259854 T2338 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4] 2> 1259854 T2338 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 1259869 T2389 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4] 2> 1259869 T2344 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4] 2> 1259869 T2373 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4] 2> 1259869 T2337 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4] 2> 1259869 T2357 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4] 2> 1260389 T2390 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
[junit4] 2> 1260389 T2390 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374439730697/collection1
[junit4] 2> 1260389 T2390 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4] 2> 1260390 T2390 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4] 2> 1260391 T2390 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4] 2> 1260392 T2390 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374439730697/collection1/'
[junit4] 2> 1260393 T2390 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374439730697/collection1/lib/README' to classloader
[junit4] 2> 1260394 T2390 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374439730697/collection1/lib/classes/' to classloader
[junit4] 2> 1260457 T2390 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4] 2> 1260525 T2390 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4] 2> 1260626 T2390 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4] 2> 1260639 T2390 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4] 2> 1261322 T2390 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4] 2> 1261337 T2390 oass.IndexSchema.readSchema default search field in schema is text
[junit4] 2> 1261341 T2390 oass.IndexSchema.readSchema unique key field: id
[junit4] 2> 1261357 T2390 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 1261363 T2390 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 1261382 T2390 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 1261384 T2390 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 1261384 T2390 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 1261385 T2390 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 1261386 T2390 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 1261386 T2390 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 1261386 T2390 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4] 2> 1261387 T2390 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374439730697/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/jetty3/
[junit4] 2> 1261387 T2390 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@71008045
[junit4] 2> 1261388 T2390 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/jetty3
[junit4] 2> 1261388 T2390 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/jetty3/index/
[junit4] 2> 1261389 T2390 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/jetty3/index' doesn't exist. Creating new index...
[junit4] 2> 1261389 T2390 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374439712923/jetty3/index
[junit4] 2> 1261393 T2390 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@2868f2da lockFactory=org.apache.lucene.store.NativeFSLockFactory@3c7f058a),segFN=segments_1,generation=1}
[junit4] 2> 1261394 T2390 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 1261400 T2390 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4] 2> 1261400 T2390 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4] 2> 1261401 T2390 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 1261402 T2390 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4] 2> 1261403 T2390 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4] 2> 1261403 T2390 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4] 2> 1261403 T2390 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4] 2> 1261404 T2390 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4] 2> 1261404 T2390 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4] 2> 1261422 T2390 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4] 2> 1261430 T2390 oass.SolrIndexSearcher.<init> Opening Searcher@374f5449 main
[junit4] 2> 1261432 T2390 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4] 2> 1261432 T2390 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4] 2> 1261437 T2391 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@374f5449 main{StandardDirectoryReader(segments_1:1)}
[junit4] 2> 1261440 T2390 oasc.CoreContainer.registerCore registering core: collection1
[junit4] 2> 1261440 T2390 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:65079/i_u/c collection:collection1 shard:shard1
[junit4] 2> 1261440 T2390 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
[junit4] 2> 1261462 T2390 oasc.ZkController.register We are http://127.0.0.1:65079/i_u/c/collection1/ and leader is http://127.0.0.1:52128/i_u/c/collection1/
[junit4] 2> 1261462 T2390 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:65079/i_u/c
[junit4] 2> 1261462 T2390 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4] 2> 1261463 T2390 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4] 2> ASYNC NEW_CORE C720 name=collection1 org.apache.solr.core.SolrCore@2a70f6c9 url=http://127.0.0.1:65079/i_u/c/collection1 node=127.0.0.1:65079_i_u%2Fc C720_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, node_name=127.0.0.1:65079_i_u%2Fc, base_url=http://127.0.0.1:65079/i_u/c}
[junit4] 2> 1261463 T2392 C720 P65079 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4] 2> 1261464 T2392 C720 P65079 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4] 2> 1261464 T2390 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1261464 T2392 C720 P65079 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4] 2> 1261464 T2392 C720 P65079 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 1261465 T2315 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0
[junit4] 2> 1261466 T2315 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4] 2> 1261466 T2392 C720 P65079 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4] 2> 1261467 T2315 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4] 2> 1261481 T2347 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true
[junit4] 2> 1261794 T2315 oejs.Server.doStart jetty-8.1.10.v201303
[...truncated too long message...]
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391)
[junit4] 2> at java.net.Socket.connect(Socket.java:579)
[junit4] 2> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
[junit4] 2> at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[junit4] 2> at java.lang.Thread.run(Thread.java:722)
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=zh_CN, timezone=America/Chihuahua
[junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_21 (64-bit)/cpus=16,threads=2,free=130353480,total=382730240
[junit4] 2> NOTE: All tests run in this JVM: [IndexSchemaTest, DistributedQueryElevationComponentTest, TestJoin, SpellingQueryConverterTest, SolrInfoMBeanTest, StressHdfsTest, QueryEqualityTest, TermsComponentTest, TestSolrQueryParser, AssignTest, TestPHPSerializedResponseWriter, UnloadDistributedZkTest, TestDynamicFieldResource, TestSolrXmlPersistence, WordBreakSolrSpellCheckerTest, LegacyHTMLStripCharFilterTest, ClusterStateUpdateTest, TestTrie, ResourceLoaderTest, ZkCLITest, TestPropInjectDefaults, TestWriterPerf, SyncSliceTest, MoreLikeThisHandlerTest, TestLMDirichletSimilarityFactory, SynonymTokenizerTest, TestReplicationHandler, ShardRoutingTest, TestSearchPerf, SuggesterTest, TestFuzzyAnalyzedSuggestions, ZkControllerTest, MinimalSchemaTest, SystemInfoHandlerTest, ShowFileRequestHandlerTest, ChaosMonkeyNothingIsSafeTest, UpdateRequestProcessorFactoryTest, TestSolrCoreProperties, TestFieldTypeCollectionResource, DirectSolrConnectionTest, AutoCommitTest, BinaryUpdateRequestHandlerTest, AddSchemaFieldsUpdateProcessorFactoryTest, SimpleFacetsTest, TestSerializedLuceneMatchVersion, TestPluginEnable, TestSolrQueryParserResource, RAMDirectoryFactoryTest, BasicFunctionalityTest, TestSort, BasicDistributedZkTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=BasicDistributedZkTest -Dtests.seed=CE9EC1F579D24215 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=zh_CN -Dtests.timezone=America/Chihuahua -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.00s J0 | BasicDistributedZkTest (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest:
[junit4] > 1) Thread[id=2759, name=recoveryCmdExecutor-1327-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
[junit4] > at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] > at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
[junit4] > at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
[junit4] > at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
[junit4] > at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391)
[junit4] > at java.net.Socket.connect(Socket.java:579)
[junit4] > at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
[junit4] > at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
[junit4] > at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
[junit4] > at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
[junit4] > at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
[junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
[junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
[junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
[junit4] > at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[junit4] > at java.lang.Thread.run(Thread.java:722)
[junit4] > at __randomizedtesting.SeedInfo.seed([CE9EC1F579D24215]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=2759, name=recoveryCmdExecutor-1327-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
[junit4] > at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] > at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
[junit4] > at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
[junit4] > at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
[junit4] > at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391)
[junit4] > at java.net.Socket.connect(Socket.java:579)
[junit4] > at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
[junit4] > at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
[junit4] > at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
[junit4] > at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
[junit4] > at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
[junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
[junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
[junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
[junit4] > at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[junit4] > at java.lang.Thread.run(Thread.java:722)
[junit4] > at __randomizedtesting.SeedInfo.seed([CE9EC1F579D24215]:0)
[junit4] Completed on J0 in 225.28s, 1 test, 2 errors <<< FAILURES!
[...truncated 570 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/build.xml:389: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/build.xml:369: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build.xml:181: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/common-build.xml:449: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/lucene/common-build.xml:1247: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/lucene/common-build.xml:890: There were test failures: 318 suites, 1342 tests, 2 suite-level errors, 698 ignored (5 assumptions)
Total time: 46 minutes 44 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure
Re: [JENKINS] Lucene-Solr-Tests-trunk-Java7 - Build # 4164 - Still Failing
Posted by Mark Miller <ma...@gmail.com>.
Dealing with this sometime today one way or another.
- Mark
On Jul 22, 2013, at 8:31 AM, Apache Jenkins Server <je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-Tests-trunk-Java7/4164/
>
> 2 tests failed.
> FAILED: junit.framework.TestSuite.org.apache.solr.cloud.BasicDistributedZkTest
>
> Error Message:
> 1 thread leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest: 1) Thread[id=3239, name=recoveryCmdExecutor-1407-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at java.net.PlainSocketImpl.socketConnect(Native Method) at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339) at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200) at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182) at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391) at java.net.Socket.connect(Socket.java:579) at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127) at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180) at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294) at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645) at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180) at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:722)
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest:
> 1) Thread[id=3239, name=recoveryCmdExecutor-1407-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
> at java.net.PlainSocketImpl.socketConnect(Native Method)
> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391)
> at java.net.Socket.connect(Socket.java:579)
> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
> at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
> at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
> at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
> at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
> at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
> at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:722)
> at __randomizedtesting.SeedInfo.seed([33D44819CFDF4DF5]:0)
>
>
> FAILED: junit.framework.TestSuite.org.apache.solr.cloud.BasicDistributedZkTest
>
> Error Message:
> There are still zombie threads that couldn't be terminated: 1) Thread[id=3239, name=recoveryCmdExecutor-1407-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at java.net.PlainSocketImpl.socketConnect(Native Method) at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339) at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200) at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182) at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391) at java.net.Socket.connect(Socket.java:579) at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127) at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180) at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294) at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645) at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180) at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:722)
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
> 1) Thread[id=3239, name=recoveryCmdExecutor-1407-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
> at java.net.PlainSocketImpl.socketConnect(Native Method)
> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391)
> at java.net.Socket.connect(Socket.java:579)
> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
> at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
> at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
> at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
> at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
> at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
> at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:722)
> at __randomizedtesting.SeedInfo.seed([33D44819CFDF4DF5]:0)
>
>
>
>
> Build Log:
> [...truncated 9682 lines...]
> [junit4] Suite: org.apache.solr.cloud.BasicDistributedZkTest
> [junit4] 2> 838345 T2789 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /rwd/ns
> [junit4] 2> 838352 T2789 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
> [junit4] 2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZkTest-1374495821781
> [junit4] 2> 838353 T2789 oasc.ZkTestServer.run STARTING ZK TEST SERVER
> [junit4] 2> 838354 T2790 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
> [junit4] 2> 838455 T2789 oasc.ZkTestServer.run start zk server on port:57977
> [junit4] 2> 838456 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
> [junit4] 2> 838460 T2796 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@20a5e53a name:ZooKeeperConnection Watcher:127.0.0.1:57977 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4] 2> 838460 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
> [junit4] 2> 838461 T2789 oascc.SolrZkClient.makePath makePath: /solr
> [junit4] 2> 838464 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
> [junit4] 2> 838473 T2798 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45dbee8d name:ZooKeeperConnection Watcher:127.0.0.1:57977/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4] 2> 838474 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
> [junit4] 2> 838474 T2789 oascc.SolrZkClient.makePath makePath: /collections/collection1
> [junit4] 2> 838478 T2789 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
> [junit4] 2> 838487 T2789 oascc.SolrZkClient.makePath makePath: /collections/control_collection
> [junit4] 2> 838489 T2789 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
> [junit4] 2> 838491 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
> [junit4] 2> 838492 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
> [junit4] 2> 838502 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
> [junit4] 2> 838503 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
> [junit4] 2> 838606 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
> [junit4] 2> 838608 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
> [junit4] 2> 838610 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
> [junit4] 2> 838611 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
> [junit4] 2> 838638 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
> [junit4] 2> 838639 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
> [junit4] 2> 838642 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
> [junit4] 2> 838643 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
> [junit4] 2> 838684 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
> [junit4] 2> 838685 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
> [junit4] 2> 838727 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
> [junit4] 2> 838728 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
> [junit4] 2> 838731 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
> [junit4] 2> 838731 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
> [junit4] 2> 839069 T2789 oejs.Server.doStart jetty-8.1.10.v20130312
> [junit4] 2> 839075 T2789 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:57982
> [junit4] 2> 839075 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
> [junit4] 2> 839076 T2789 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
> [junit4] 2> 839077 T2789 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225
> [junit4] 2> 839077 T2789 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/'
> [junit4] 2> 839115 T2789 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/solr.xml
> [junit4] 2> 839206 T2789 oasc.CoreContainer.<init> New CoreContainer 454265048
> [junit4] 2> 839207 T2789 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/]
> [junit4] 2> 839209 T2789 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
> [junit4] 2> 839209 T2789 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
> [junit4] 2> 839210 T2789 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
> [junit4] 2> 839210 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
> [junit4] 2> 839211 T2789 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
> [junit4] 2> 839211 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
> [junit4] 2> 839212 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
> [junit4] 2> 839212 T2789 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
> [junit4] 2> 839213 T2789 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
> [junit4] 2> 839214 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
> [junit4] 2> 839230 T2789 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
> [junit4] 2> 839231 T2789 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57977/solr
> [junit4] 2> 839231 T2789 oasc.ZkController.checkChrootPath zkHost includes chroot
> [junit4] 2> 839232 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
> [junit4] 2> 839239 T2809 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a5da037 name:ZooKeeperConnection Watcher:127.0.0.1:57977 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4] 2> 839239 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
> [junit4] 2> 839241 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
> [junit4] 2> 839252 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
> [junit4] 2> 839257 T2811 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6b45e390 name:ZooKeeperConnection Watcher:127.0.0.1:57977/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4] 2> 839257 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
> [junit4] 2> 839259 T2789 oascc.SolrZkClient.makePath makePath: /overseer/queue
> [junit4] 2> 839270 T2789 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
> [junit4] 2> 839273 T2789 oascc.SolrZkClient.makePath makePath: /live_nodes
> [junit4] 2> 839275 T2789 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57982_rwd%2Fns
> [junit4] 2> 839283 T2789 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57982_rwd%2Fns
> [junit4] 2> 839286 T2789 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
> [junit4] 2> 839301 T2789 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
> [junit4] 2> 839309 T2789 oasc.Overseer.start Overseer (id=90078958176763907-127.0.0.1:57982_rwd%2Fns-n_0000000000) starting
> [junit4] 2> 839323 T2789 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
> [junit4] 2> 839326 T2813 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
> [junit4] 2> 839327 T2789 oascc.SolrZkClient.makePath makePath: /clusterstate.json
> [junit4] 2> 839329 T2789 oascc.SolrZkClient.makePath makePath: /aliases.json
> [junit4] 2> 839331 T2789 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
> [junit4] 2> 839333 T2812 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
> [junit4] 2> 839338 T2814 oasc.ZkController.publish publishing core=collection1 state=down
> [junit4] 2> 839338 T2814 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
> [junit4] 2> 839340 T2814 oasc.ZkController.waitForCoreNodeName look for our core node name
> [junit4] 2> 840837 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
> [junit4] 2> 840838 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
> [junit4] 2> "operation":"state",
> [junit4] 2> "core_node_name":"core_node1",
> [junit4] 2> "numShards":"1",
> [junit4] 2> "shard":null,
> [junit4] 2> "roles":null,
> [junit4] 2> "state":"down",
> [junit4] 2> "shard_state":"active",
> [junit4] 2> "core":"collection1",
> [junit4] 2> "collection":"control_collection",
> [junit4] 2> "shard_range":null,
> [junit4] 2> "node_name":"127.0.0.1:57982_rwd%2Fns",
> [junit4] 2> "base_url":"http://127.0.0.1:57982/rwd/ns"}
> [junit4] 2> 840838 T2812 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
> [junit4] 2> 840839 T2812 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
> [junit4] 2> 840855 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
> [junit4] 2> 841342 T2814 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
> [junit4] 2> 841342 T2814 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/collection1
> [junit4] 2> 841342 T2814 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
> [junit4] 2> 841343 T2814 oasc.ZkController.createCollectionZkNode Collection zkNode exists
> [junit4] 2> 841343 T2814 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
> [junit4] 2> 841344 T2814 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/collection1/'
> [junit4] 2> 841346 T2814 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/collection1/lib/README' to classloader
> [junit4] 2> 841347 T2814 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/collection1/lib/classes/' to classloader
> [junit4] 2> 841406 T2814 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
> [junit4] 2> 841470 T2814 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
> [junit4] 2> 841571 T2814 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
> [junit4] 2> 841583 T2814 oass.IndexSchema.readSchema [collection1] Schema name=test
> [junit4] 2> 842242 T2814 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
> [junit4] 2> 842256 T2814 oass.IndexSchema.readSchema default search field in schema is text
> [junit4] 2> 842260 T2814 oass.IndexSchema.readSchema unique key field: id
> [junit4] 2> 842273 T2814 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
> [junit4] 2> 842278 T2814 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
> [junit4] 2> 842283 T2814 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
> [junit4] 2> 842284 T2814 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
> [junit4] 2> 842284 T2814 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
> [junit4] 2> 842285 T2814 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
> [junit4] 2> 842286 T2814 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
> [junit4] 2> 842286 T2814 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
> [junit4] 2> 842286 T2814 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
> [junit4] 2> 842287 T2814 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/control/data/
> [junit4] 2> 842287 T2814 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5de17399
> [junit4] 2> 842288 T2814 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/control/data
> [junit4] 2> 842289 T2814 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/control/data/index/
> [junit4] 2> 842289 T2814 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/control/data/index' doesn't exist. Creating new index...
> [junit4] 2> 842289 T2814 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/control/data/index
> [junit4] 2> 842294 T2814 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
> [junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@203e9b48 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7e3b398a),segFN=segments_1,generation=1}
> [junit4] 2> 842295 T2814 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
> [junit4] 2> 842300 T2814 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
> [junit4] 2> 842300 T2814 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
> [junit4] 2> 842301 T2814 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
> [junit4] 2> 842302 T2814 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
> [junit4] 2> 842303 T2814 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
> [junit4] 2> 842303 T2814 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
> [junit4] 2> 842303 T2814 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
> [junit4] 2> 842304 T2814 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
> [junit4] 2> 842304 T2814 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
> [junit4] 2> 842321 T2814 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
> [junit4] 2> 842329 T2814 oass.SolrIndexSearcher.<init> Opening Searcher@52ea3d41 main
> [junit4] 2> 842330 T2814 oasu.CommitTracker.<init> Hard AutoCommit: disabled
> [junit4] 2> 842331 T2814 oasu.CommitTracker.<init> Soft AutoCommit: disabled
> [junit4] 2> 842334 T2815 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@52ea3d41 main{StandardDirectoryReader(segments_1:1)}
> [junit4] 2> 842337 T2814 oasc.CoreContainer.registerCore registering core: collection1
> [junit4] 2> 842337 T2814 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57982/rwd/ns collection:control_collection shard:shard1
> [junit4] 2> 842337 T2814 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
> [junit4] 2> 842356 T2814 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
> [junit4] 2> 842377 T2814 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
> [junit4] 2> 842388 T2814 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
> [junit4] 2> 842388 T2814 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
> [junit4] 2> 842388 T2814 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57982/rwd/ns/collection1/
> [junit4] 2> 842389 T2814 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
> [junit4] 2> 842389 T2814 oasc.SyncStrategy.syncToMe http://127.0.0.1:57982/rwd/ns/collection1/ has no replicas
> [junit4] 2> 842389 T2814 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57982/rwd/ns/collection1/ shard1
> [junit4] 2> 842390 T2814 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
> [junit4] 2> 843864 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
> [junit4] 2> 843888 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
> [junit4] 2> 843925 T2814 oasc.ZkController.register We are http://127.0.0.1:57982/rwd/ns/collection1/ and leader is http://127.0.0.1:57982/rwd/ns/collection1/
> [junit4] 2> 843925 T2814 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57982/rwd/ns
> [junit4] 2> 843925 T2814 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
> [junit4] 2> 843926 T2814 oasc.ZkController.publish publishing core=collection1 state=active
> [junit4] 2> 843926 T2814 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
> [junit4] 2> 843928 T2814 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
> [junit4] 2> 843929 T2789 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0
> [junit4] 2> 843930 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
> [junit4] 2> 843930 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
> [junit4] 2> 843943 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:
> [junit4] 2> 843944 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
> [junit4] 2> 843947 T2818 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e187f0c name:ZooKeeperConnection Watcher:127.0.0.1:57977/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4] 2> 843947 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
> [junit4] 2> 843948 T2789 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
> [junit4] 2> 843950 T2789 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
> [junit4] 2> 844271 T2789 oejs.Server.doStart jetty-8.1.10.v20130312
> [junit4] 2> 844275 T2789 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:57986
> [junit4] 2> 844275 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
> [junit4] 2> 844276 T2789 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
> [junit4] 2> 844277 T2789 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378
> [junit4] 2> 844277 T2789 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/'
> [junit4] 2> 844317 T2789 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/solr.xml
> [junit4] 2> 844409 T2789 oasc.CoreContainer.<init> New CoreContainer 1420732659
> [junit4] 2> 844410 T2789 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/]
> [junit4] 2> 844412 T2789 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
> [junit4] 2> 844413 T2789 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
> [junit4] 2> 844413 T2789 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
> [junit4] 2> 844414 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
> [junit4] 2> 844414 T2789 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
> [junit4] 2> 844415 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
> [junit4] 2> 844415 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
> [junit4] 2> 844416 T2789 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
> [junit4] 2> 844417 T2789 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
> [junit4] 2> 844417 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
> [junit4] 2> 844433 T2789 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
> [junit4] 2> 844434 T2789 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57977/solr
> [junit4] 2> 844434 T2789 oasc.ZkController.checkChrootPath zkHost includes chroot
> [junit4] 2> 844436 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
> [junit4] 2> 844444 T2829 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4741fcc1 name:ZooKeeperConnection Watcher:127.0.0.1:57977 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4] 2> 844444 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
> [junit4] 2> 844447 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
> [junit4] 2> 844458 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
> [junit4] 2> 844460 T2831 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c0ab1f9 name:ZooKeeperConnection Watcher:127.0.0.1:57977/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4] 2> 844461 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
> [junit4] 2> 844474 T2789 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
> [junit4] 2> 845392 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
> [junit4] 2> 845393 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
> [junit4] 2> "operation":"state",
> [junit4] 2> "core_node_name":"core_node1",
> [junit4] 2> "numShards":"1",
> [junit4] 2> "shard":"shard1",
> [junit4] 2> "roles":null,
> [junit4] 2> "state":"active",
> [junit4] 2> "shard_state":"active",
> [junit4] 2> "core":"collection1",
> [junit4] 2> "collection":"control_collection",
> [junit4] 2> "shard_range":null,
> [junit4] 2> "node_name":"127.0.0.1:57982_rwd%2Fns",
> [junit4] 2> "base_url":"http://127.0.0.1:57982/rwd/ns"}
> [junit4] 2> 845436 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
> [junit4] 2> 845436 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
> [junit4] 2> 845436 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
> [junit4] 2> 845478 T2789 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57986_rwd%2Fns
> [junit4] 2> 845480 T2789 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57986_rwd%2Fns
> [junit4] 2> 845482 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
> [junit4] 2> 845482 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
> [junit4] 2> 845482 T2831 oascc.ZkStateReader$3.process Updating live nodes... (2)
> [junit4] 2> 845483 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
> [junit4] 2> 845483 T2818 oascc.ZkStateReader$3.process Updating live nodes... (2)
> [junit4] 2> 845491 T2811 oascc.ZkStateReader$3.process Updating live nodes... (2)
> [junit4] 2> 845498 T2832 oasc.ZkController.publish publishing core=collection1 state=down
> [junit4] 2> 845499 T2832 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
> [junit4] 2> 845500 T2832 oasc.ZkController.waitForCoreNodeName look for our core node name
> [junit4] 2> 846940 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
> [junit4] 2> 846942 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
> [junit4] 2> "operation":"state",
> [junit4] 2> "core_node_name":"core_node1",
> [junit4] 2> "numShards":"2",
> [junit4] 2> "shard":null,
> [junit4] 2> "roles":null,
> [junit4] 2> "state":"down",
> [junit4] 2> "shard_state":"active",
> [junit4] 2> "core":"collection1",
> [junit4] 2> "collection":"collection1",
> [junit4] 2> "shard_range":null,
> [junit4] 2> "node_name":"127.0.0.1:57986_rwd%2Fns",
> [junit4] 2> "base_url":"http://127.0.0.1:57986/rwd/ns"}
> [junit4] 2> 846942 T2812 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 2
> [junit4] 2> 846942 T2812 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
> [junit4] 2> 846959 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4] 2> 846959 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4] 2> 846959 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4] 2> 847502 T2832 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
> [junit4] 2> 847502 T2832 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/collection1
> [junit4] 2> 847502 T2832 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
> [junit4] 2> 847503 T2832 oasc.ZkController.createCollectionZkNode Collection zkNode exists
> [junit4] 2> 847503 T2832 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
> [junit4] 2> 847504 T2832 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/collection1/'
> [junit4] 2> 847506 T2832 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/collection1/lib/classes/' to classloader
> [junit4] 2> 847507 T2832 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/collection1/lib/README' to classloader
> [junit4] 2> 847559 T2832 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
> [junit4] 2> 847604 T2832 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
> [junit4] 2> 847706 T2832 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
> [junit4] 2> 847715 T2832 oass.IndexSchema.readSchema [collection1] Schema name=test
> [junit4] 2> 848185 T2832 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
> [junit4] 2> 848195 T2832 oass.IndexSchema.readSchema default search field in schema is text
> [junit4] 2> 848198 T2832 oass.IndexSchema.readSchema unique key field: id
> [junit4] 2> 848229 T2832 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
> [junit4] 2> 848233 T2832 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
> [junit4] 2> 848237 T2832 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
> [junit4] 2> 848237 T2832 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
> [junit4] 2> 848237 T2832 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
> [junit4] 2> 848238 T2832 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
> [junit4] 2> 848238 T2832 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
> [junit4] 2> 848239 T2832 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
> [junit4] 2> 848239 T2832 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
> [junit4] 2> 848239 T2832 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty1/
> [junit4] 2> 848239 T2832 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5de17399
> [junit4] 2> 848240 T2832 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty1
> [junit4] 2> 848240 T2832 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty1/index/
> [junit4] 2> 848241 T2832 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty1/index' doesn't exist. Creating new index...
> [junit4] 2> 848241 T2832 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty1/index
> [junit4] 2> 848245 T2832 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
> [junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@68ad8c59 lockFactory=org.apache.lucene.store.NativeFSLockFactory@581f222d),segFN=segments_1,generation=1}
> [junit4] 2> 848245 T2832 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
> [junit4] 2> 848249 T2832 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
> [junit4] 2> 848250 T2832 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
> [junit4] 2> 848250 T2832 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
> [junit4] 2> 848251 T2832 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
> [junit4] 2> 848251 T2832 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
> [junit4] 2> 848252 T2832 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
> [junit4] 2> 848252 T2832 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
> [junit4] 2> 848252 T2832 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
> [junit4] 2> 848253 T2832 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
> [junit4] 2> 848265 T2832 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
> [junit4] 2> 848271 T2832 oass.SolrIndexSearcher.<init> Opening Searcher@7e54a378 main
> [junit4] 2> 848272 T2832 oasu.CommitTracker.<init> Hard AutoCommit: disabled
> [junit4] 2> 848272 T2832 oasu.CommitTracker.<init> Soft AutoCommit: disabled
> [junit4] 2> 848276 T2833 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7e54a378 main{StandardDirectoryReader(segments_1:1)}
> [junit4] 2> 848277 T2832 oasc.CoreContainer.registerCore registering core: collection1
> [junit4] 2> 848277 T2832 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57986/rwd/ns collection:collection1 shard:shard1
> [junit4] 2> 848278 T2832 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
> [junit4] 2> 848293 T2832 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
> [junit4] 2> 848315 T2832 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
> [junit4] 2> 848326 T2832 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
> [junit4] 2> 848326 T2832 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
> [junit4] 2> 848326 T2832 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57986/rwd/ns/collection1/
> [junit4] 2> 848327 T2832 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
> [junit4] 2> 848327 T2832 oasc.SyncStrategy.syncToMe http://127.0.0.1:57986/rwd/ns/collection1/ has no replicas
> [junit4] 2> 848327 T2832 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57986/rwd/ns/collection1/ shard1
> [junit4] 2> 848328 T2832 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
> [junit4] 2> 848463 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
> [junit4] 2> 848480 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4] 2> 848480 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4] 2> 848480 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4] 2> 848487 T2832 oasc.ZkController.register We are http://127.0.0.1:57986/rwd/ns/collection1/ and leader is http://127.0.0.1:57986/rwd/ns/collection1/
> [junit4] 2> 848487 T2832 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57986/rwd/ns
> [junit4] 2> 848487 T2832 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
> [junit4] 2> 848487 T2832 oasc.ZkController.publish publishing core=collection1 state=active
> [junit4] 2> 848488 T2832 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
> [junit4] 2> 848498 T2832 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
> [junit4] 2> 848499 T2789 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0
> [junit4] 2> 848500 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
> [junit4] 2> 848500 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
> [junit4] 2> 848808 T2789 oejs.Server.doStart jetty-8.1.10.v20130312
> [junit4] 2> 848812 T2789 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:57989
> [junit4] 2> 848812 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
> [junit4] 2> 848813 T2789 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
> [junit4] 2> 848813 T2789 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937
> [junit4] 2> 848814 T2789 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/'
> [junit4] 2> 848856 T2789 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/solr.xml
> [junit4] 2> 848958 T2789 oasc.CoreContainer.<init> New CoreContainer 1108024381
> [junit4] 2> 848959 T2789 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/]
> [junit4] 2> 848961 T2789 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
> [junit4] 2> 848962 T2789 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
> [junit4] 2> 848962 T2789 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
> [junit4] 2> 848963 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
> [junit4] 2> 848964 T2789 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
> [junit4] 2> 848964 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
> [junit4] 2> 848965 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
> [junit4] 2> 848965 T2789 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
> [junit4] 2> 848966 T2789 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
> [junit4] 2> 848966 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
> [junit4] 2> 848986 T2789 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
> [junit4] 2> 848988 T2789 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57977/solr
> [junit4] 2> 848988 T2789 oasc.ZkController.checkChrootPath zkHost includes chroot
> [junit4] 2> 848989 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
> [junit4] 2> 849032 T2845 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@33e605f4 name:ZooKeeperConnection Watcher:127.0.0.1:57977 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4] 2> 849033 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
> [junit4] 2> 849067 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
> [junit4] 2> 849081 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
> [junit4] 2> 849084 T2847 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2119f463 name:ZooKeeperConnection Watcher:127.0.0.1:57977/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4] 2> 849084 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
> [junit4] 2> 849138 T2789 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
> [junit4] 2> 849986 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
> [junit4] 2> 849988 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
> [junit4] 2> "operation":"state",
> [junit4] 2> "core_node_name":"core_node1",
> [junit4] 2> "numShards":"2",
> [junit4] 2> "shard":"shard1",
> [junit4] 2> "roles":null,
> [junit4] 2> "state":"active",
> [junit4] 2> "shard_state":"active",
> [junit4] 2> "core":"collection1",
> [junit4] 2> "collection":"collection1",
> [junit4] 2> "shard_range":null,
> [junit4] 2> "node_name":"127.0.0.1:57986_rwd%2Fns",
> [junit4] 2> "base_url":"http://127.0.0.1:57986/rwd/ns"}
> [junit4] 2> 850020 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4] 2> 850020 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4] 2> 850020 T2847 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4] 2> 850020 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
> [junit4] 2> 850142 T2789 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57989_rwd%2Fns
> [junit4] 2> 850171 T2789 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57989_rwd%2Fns
> [junit4] 2> 850173 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
> [junit4] 2> 850173 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
> [junit4] 2> 850174 T2847 oascc.ZkStateReader$3.process Updating live nodes... (3)
> [junit4] 2> 850174 T2831 oascc.ZkStateReader$3.process Updating live nodes... (3)
> [junit4] 2> 850174 T2847 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 850174 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 850175 T2818 oascc.ZkStateReader$3.process Updating live nodes... (3)
> [junit4] 2> 850188 T2811 oascc.ZkStateReader$3.process Updating live nodes... (3)
> [junit4] 2> 850195 T2848 oasc.ZkController.publish publishing core=collection1 state=down
> [junit4] 2> 850195 T2848 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
> [junit4] 2> 850197 T2848 oasc.ZkController.waitForCoreNodeName look for our core node name
> [junit4] 2> 851532 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
> [junit4] 2> 851533 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
> [junit4] 2> "operation":"state",
> [junit4] 2> "core_node_name":"core_node2",
> [junit4] 2> "numShards":"2",
> [junit4] 2> "shard":null,
> [junit4] 2> "roles":null,
> [junit4] 2> "state":"down",
> [junit4] 2> "shard_state":"active",
> [junit4] 2> "core":"collection1",
> [junit4] 2> "collection":"collection1",
> [junit4] 2> "shard_range":null,
> [junit4] 2> "node_name":"127.0.0.1:57989_rwd%2Fns",
> [junit4] 2> "base_url":"http://127.0.0.1:57989/rwd/ns"}
> [junit4] 2> 851534 T2812 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
> [junit4] 2> 851534 T2812 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
> [junit4] 2> 851551 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 851551 T2847 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 851551 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 851551 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 852199 T2848 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
> [junit4] 2> 852199 T2848 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/collection1
> [junit4] 2> 852200 T2848 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
> [junit4] 2> 852200 T2848 oasc.ZkController.createCollectionZkNode Collection zkNode exists
> [junit4] 2> 852201 T2848 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
> [junit4] 2> 852201 T2848 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/collection1/'
> [junit4] 2> 852203 T2848 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/collection1/lib/README' to classloader
> [junit4] 2> 852204 T2848 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/collection1/lib/classes/' to classloader
> [junit4] 2> 852262 T2848 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
> [junit4] 2> 852332 T2848 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
> [junit4] 2> 852434 T2848 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
> [junit4] 2> 852447 T2848 oass.IndexSchema.readSchema [collection1] Schema name=test
> [junit4] 2> 853099 T2848 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
> [junit4] 2> 853109 T2848 oass.IndexSchema.readSchema default search field in schema is text
> [junit4] 2> 853112 T2848 oass.IndexSchema.readSchema unique key field: id
> [junit4] 2> 853121 T2848 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
> [junit4] 2> 853125 T2848 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
> [junit4] 2> 853128 T2848 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
> [junit4] 2> 853129 T2848 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
> [junit4] 2> 853129 T2848 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
> [junit4] 2> 853129 T2848 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
> [junit4] 2> 853130 T2848 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
> [junit4] 2> 853130 T2848 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
> [junit4] 2> 853130 T2848 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
> [junit4] 2> 853131 T2848 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty2/
> [junit4] 2> 853131 T2848 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5de17399
> [junit4] 2> 853132 T2848 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty2
> [junit4] 2> 853132 T2848 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty2/index/
> [junit4] 2> 853132 T2848 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty2/index' doesn't exist. Creating new index...
> [junit4] 2> 853132 T2848 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty2/index
> [junit4] 2> 853135 T2848 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
> [junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5611756f lockFactory=org.apache.lucene.store.NativeFSLockFactory@314b85a),segFN=segments_1,generation=1}
> [junit4] 2> 853136 T2848 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
> [junit4] 2> 853140 T2848 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
> [junit4] 2> 853140 T2848 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
> [junit4] 2> 853141 T2848 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
> [junit4] 2> 853141 T2848 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
> [junit4] 2> 853142 T2848 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
> [junit4] 2> 853142 T2848 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
> [junit4] 2> 853142 T2848 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
> [junit4] 2> 853143 T2848 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
> [junit4] 2> 853143 T2848 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
> [junit4] 2> 853156 T2848 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
> [junit4] 2> 853163 T2848 oass.SolrIndexSearcher.<init> Opening Searcher@6fcd2a55 main
> [junit4] 2> 853164 T2848 oasu.CommitTracker.<init> Hard AutoCommit: disabled
> [junit4] 2> 853164 T2848 oasu.CommitTracker.<init> Soft AutoCommit: disabled
> [junit4] 2> 853167 T2849 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6fcd2a55 main{StandardDirectoryReader(segments_1:1)}
> [junit4] 2> 853169 T2848 oasc.CoreContainer.registerCore registering core: collection1
> [junit4] 2> 853169 T2848 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57989/rwd/ns collection:collection1 shard:shard2
> [junit4] 2> 853169 T2848 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
> [junit4] 2> 853180 T2848 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
> [junit4] 2> 853196 T2848 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
> [junit4] 2> 853204 T2848 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
> [junit4] 2> 853204 T2848 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
> [junit4] 2> 853204 T2848 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57989/rwd/ns/collection1/
> [junit4] 2> 853204 T2848 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
> [junit4] 2> 853205 T2848 oasc.SyncStrategy.syncToMe http://127.0.0.1:57989/rwd/ns/collection1/ has no replicas
> [junit4] 2> 853205 T2848 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57989/rwd/ns/collection1/ shard2
> [junit4] 2> 853205 T2848 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
> [junit4] 2> 854558 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
> [junit4] 2> 854601 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 854601 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 854601 T2847 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 854601 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 854645 T2848 oasc.ZkController.register We are http://127.0.0.1:57989/rwd/ns/collection1/ and leader is http://127.0.0.1:57989/rwd/ns/collection1/
> [junit4] 2> 854645 T2848 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57989/rwd/ns
> [junit4] 2> 854645 T2848 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
> [junit4] 2> 854646 T2848 oasc.ZkController.publish publishing core=collection1 state=active
> [junit4] 2> 854646 T2848 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
> [junit4] 2> 854648 T2848 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
> [junit4] 2> 854649 T2789 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0
> [junit4] 2> 854650 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
> [junit4] 2> 854650 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
> [junit4] 2> 854970 T2789 oejs.Server.doStart jetty-8.1.10.v20130312
> [junit4] 2> 854974 T2789 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:57992
> [junit4] 2> 854975 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
> [junit4] 2> 854976 T2789 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
> [junit4] 2> 854976 T2789 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091
> [junit4] 2> 854977 T2789 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/'
> [junit4] 2> 855022 T2789 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/solr.xml
> [junit4] 2> 855122 T2789 oasc.CoreContainer.<init> New CoreContainer 1041480000
> [junit4] 2> 855123 T2789 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/]
> [junit4] 2> 855125 T2789 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
> [junit4] 2> 855126 T2789 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
> [junit4] 2> 855126 T2789 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
> [junit4] 2> 855127 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
> [junit4] 2> 855127 T2789 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
> [junit4] 2> 855128 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
> [junit4] 2> 855128 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
> [junit4] 2> 855129 T2789 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
> [junit4] 2> 855129 T2789 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
> [junit4] 2> 855130 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
> [junit4] 2> 855147 T2789 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
> [junit4] 2> 855148 T2789 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57977/solr
> [junit4] 2> 855148 T2789 oasc.ZkController.checkChrootPath zkHost includes chroot
> [junit4] 2> 855149 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
> [junit4] 2> 855152 T2861 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7ad6e636 name:ZooKeeperConnection Watcher:127.0.0.1:57977 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4] 2> 855153 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
> [junit4] 2> 855166 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
> [junit4] 2> 855176 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
> [junit4] 2> 855178 T2863 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@55a547bc name:ZooKeeperConnection Watcher:127.0.0.1:57977/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
> [junit4] 2> 855179 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
> [junit4] 2> 855190 T2789 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
> [junit4] 2> 856115 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
> [junit4] 2> 856116 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
> [junit4] 2> "operation":"state",
> [junit4] 2> "core_node_name":"core_node2",
> [junit4] 2> "numShards":"2",
> [junit4] 2> "shard":"shard2",
> [junit4] 2> "roles":null,
> [junit4] 2> "state":"active",
> [junit4] 2> "shard_state":"active",
> [junit4] 2> "core":"collection1",
> [junit4] 2> "collection":"collection1",
> [junit4] 2> "shard_range":null,
> [junit4] 2> "node_name":"127.0.0.1:57989_rwd%2Fns",
> [junit4] 2> "base_url":"http://127.0.0.1:57989/rwd/ns"}
> [junit4] 2> 856152 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 856152 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 856152 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 856152 T2863 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 856152 T2847 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 856193 T2789 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57992_rwd%2Fns
> [junit4] 2> 856195 T2789 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57992_rwd%2Fns
> [junit4] 2> 856197 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 856197 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
> [junit4] 2> 856198 T2847 oascc.ZkStateReader$3.process Updating live nodes... (4)
> [junit4] 2> 856198 T2863 oascc.ZkStateReader$3.process Updating live nodes... (4)
> [junit4] 2> 856198 T2831 oascc.ZkStateReader$3.process Updating live nodes... (4)
> [junit4] 2> 856199 T2863 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
> [junit4] 2> 856198 T2847 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
> [junit4] 2> 856199 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
> [junit4] 2> 856207 T2818 oascc.ZkStateReader$3.process Updating live nodes... (4)
> [junit4] 2> 856207 T2811 oascc.ZkStateReader$3.process Updating live nodes... (4)
> [junit4] 2> 856214 T2864 oasc.ZkController.publish publishing core=collection1 state=down
> [junit4] 2> 856214 T2864 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
> [junit4] 2> 856216 T2864 oasc.ZkController.waitForCoreNodeName look for our core node name
> [junit4] 2> 857656 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
> [junit4] 2> 857657 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
> [junit4] 2> "operation":"state",
> [junit4] 2> "core_node_name":"core_node3",
> [junit4] 2> "numShards":"2",
> [junit4] 2> "shard":null,
> [junit4] 2> "roles":null,
> [junit4] 2> "state":"down",
> [junit4] 2> "shard_state":"active",
> [junit4] 2> "core":"collection1",
> [junit4] 2> "collection":"collection1",
> [junit4] 2> "shard_range":null,
> [junit4] 2> "node_name":"127.0.0.1:57992_rwd%2Fns",
> [junit4] 2> "base_url":"http://127.0.0.1:57992/rwd/ns"}
> [junit4] 2> 857658 T2812 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
> [junit4] 2> 857658 T2812 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
> [junit4] 2> 857707 T2863 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4] 2> 857707 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4] 2> 857707 T2847 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4] 2> 857707 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4] 2> 857707 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
> [junit4] 2> 858218 T2864 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
> [junit4] 2> 858218 T2864 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/collection1
> [junit4] 2> 858218 T2864 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
> [junit4] 2> 858219 T2864 oasc.ZkController.createCollectionZkNode Collection zkNode exists
> [junit4] 2> 858219 T2864 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
> [junit4] 2> 858220 T2864 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/collection1/'
> [junit4] 2> 858222 T2864 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/collection1/lib/README' to classloader
> [junit4] 2> 858222 T2864 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/collection1/lib/classes/' to classloader
> [junit4] 2> 858291 T2864 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
> [junit4] 2> 858338 T2864 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
> [junit4] 2> 858440 T2864 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
> [junit4] 2> 858449 T2864 oass.IndexSchema.readSchema [collection1] Schema name=test
> [junit4] 2> 859063 T2864 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
> [junit4] 2> 859074 T2864 oass.IndexSchema.readSchema default search field in schema is text
> [junit4] 2> 859077 T2864 oass.IndexSchema.readSchema unique key field: id
> [junit4] 2> 859088 T2864 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
> [junit4] 2> 859092 T2864 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
> [junit4] 2> 859096 T2864 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
> [junit4] 2> 859097 T2864 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
> [junit4] 2> 859097 T2864 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
> [junit4] 2> 859097 T2864 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
> [junit4] 2> 859098 T2864 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
> [junit4] 2> 859099 T2864 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
> [junit4] 2> 859099 T2864 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
> [junit4] 2> 859099 T2864 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty3/
> [junit4] 2> 859099 T2864 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5de17399
> [junit4] 2> 859101 T2864 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty3
> [junit4] 2> 859101 T2864 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty3/index/
> [junit4] 2> 859102 T2864 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty3/index' doesn't exist. Creating new index...
> [junit4] 2> 859102 T2864 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty3/index
> [junit4] 2> 859105 T2864 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
> [junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@24446f84 lockFactory=org.apache.lucene.store.NativeFSLockFactory@bb29558),segFN=segments_1,generation=1}
> [junit4] 2> 859106 T2864 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
> [junit4] 2> 859110 T2864 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
> [junit4] 2> 859110 T2864 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
> [junit4] 2> 859111 T2864 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
> [junit4] 2> 859112 T2864 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
> [junit4] 2> 859112 T2864 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
> [junit4] 2> 859112 T2864 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
> [junit4] 2> 859113 T2864 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
> [junit4] 2> 859113 T2864 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
> [junit4] 2> 859113 T2864 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
> [junit4] 2> 859128 T2864 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
> [junit4] 2> 859136 T2864 oass.SolrIndexSearcher.<init> Opening Searcher@67e5d2fa main
> [junit4] 2> 859137 T2864 oasu.CommitTracker.<init> Hard AutoCommit: disabled
> [junit4] 2> 859137 T2864 oasu.CommitTracker.<init> Soft AutoCommit: disabled
> [junit4] 2> 859140 T2865 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@67e5d2fa main{StandardDirectoryReader(segments_1:1)}
> [junit4] 2> 859142 T2864 oasc.CoreContainer.registerCore registering core: collection1
> [junit4] 2> 859142 T2864 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57992/rwd/ns collection:collection1 shard:shard1
> [junit4] 2> 859142 T2864 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
> [junit4] 2> 859165 T2864 oasc.ZkController.register We are http://127.0.0.1:57992/rwd/ns/collection1/ and leader is http://127.0.0.1:57986/rwd/ns/collection1/
> [junit4] 2> 859165 T2864 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57992/rwd/ns
> [junit4] 2> 859165 T2864 oasc.ZkController.checkRecovery Core needs to recover:collection1
> [junit4] 2> 859165 T2864 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
> [junit4] 2> ASYNC NEW_CORE C624 name=collection1 org.apache.solr.core.SolrCore@b98d041 url=http://127.0.0.1:57992/rwd/ns/collection1 node=127.0.0.1:57992_rwd%2Fns C624_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, node_name=127.0.0.1:57992_rwd%2Fns, base_url=http://127.0.0.1:57992/rwd/ns}
> [junit4] 2> 859166 T2866 C624 P57992 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
> [junit4] 2> 859166 T2864 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
> [junit4] 2> 859166 T2866 C624 P57992 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
> [junit4] 2> 859167 T2866 C624 P57992 oasc.ZkController.publish publishing core=collection1 state=recovering
> [junit4] 2> 859167 T2866 C624 P57992 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
> [junit4] 2> 859167 T2789 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0
> [junit4] 2> 859168 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
> [junit4] 2> 859169 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
> [junit4] 2> 859173 T2866 C624 P57992 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
> [junit4] 2> 859185 T2821 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true
> [junit4] 2> 859211 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
> [junit4] 2> 859211 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
> [junit4] 2> "operation":"state",
> [junit4] 2> "core_node_name":"core_node3",
> [junit4] 2> "numShards":"2",
> [junit4] 2> "shard":"shard1",
> [junit4] 2> "roles":null,
> [junit4] 2> "state":"recovering",
> [junit4] 2> "shard_state":"active",
> [junit4] 2> "core":"collection1",
> [ju
>
> [...truncated too long message...]
>
> ry doPackFST= true), range_facet_sl=PostingsFormat(name=Memory doPackFST= true), range_facet_si=PostingsFormat(name=MockRandom), oddField_s=PostingsFormat(name=Memory doPackFST= true), sequence_i=PostingsFormat(name=Memory doPackFST= true), name=PostingsFormat(name=Memory doPackFST= true), foo_i=PostingsFormat(name=TestBloomFilteredLucene41Postings), regex_dup_B_s=PostingsFormat(name=MockRandom), multiDefault=PostingsFormat(name=MockRandom), n_tf1=PostingsFormat(name=MockRandom), n_dt1=PostingsFormat(name=TestBloomFilteredLucene41Postings), genre_s=PostingsFormat(name=Memory doPackFST= true), author_t=PostingsFormat(name=TestBloomFilteredLucene41Postings), range_facet_l=PostingsFormat(name=MockRandom), n_ti1=PostingsFormat(name=Memory doPackFST= true), text=Pulsing41(freqCutoff=4 minBlockSize=87 maxBlockSize=218), _version_=PostingsFormat(name=TestBloomFilteredLucene41Postings), val_i=PostingsFormat(name=Memory doPackFST= true), SubjectTerms_mfacet=PostingsFormat(name=TestBloomFilteredLucene41Postings), series_t=PostingsFormat(name=TestBloomFilteredLucene41Postings), a_t=PostingsFormat(name=MockRandom), n_tdt1=PostingsFormat(name=TestBloomFilteredLucene41Postings), regex_dup_A_s=Pulsing41(freqCutoff=4 minBlockSize=87 maxBlockSize=218), price=Pulsing41(freqCutoff=4 minBlockSize=87 maxBlockSize=218), other_tl1=PostingsFormat(name=TestBloomFilteredLucene41Postings), n_l1=PostingsFormat(name=MockRandom), a_si=PostingsFormat(name=TestBloomFilteredLucene41Postings), inStock=Pulsing41(freqCutoff=4 minBlockSize=87 maxBlockSize=218)}, docValues:{timestamp=DocValuesFormat(name=Lucene42)}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=ar_BH, timezone=Europe/Helsinki
> [junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_21 (64-bit)/cpus=16,threads=2,free=238764256,total=399507456
> [junit4] 2> NOTE: All tests run in this JVM: [TestPerFieldSimilarity, TestQueryUtils, TestSurroundQueryParser, QueryEqualityTest, PeerSyncTest, SolrTestCaseJ4Test, TestPropInjectDefaults, RecoveryZkTest, TestStressRecovery, UpdateRequestProcessorFactoryTest, QueryElevationComponentTest, TestSolrXml, StatelessScriptUpdateProcessorFactoryTest, ScriptEngineTest, TestSchemaNameResource, TestSerializedLuceneMatchVersion, TestRandomDVFaceting, PolyFieldTest, RegexBoostProcessorTest, AutoCommitTest, TestFastLRUCache, LegacyHTMLStripCharFilterTest, ZkCLITest, AddSchemaFieldsUpdateProcessorFactoryTest, DateFieldTest, AssignTest, SOLR749Test, TermsComponentTest, TestSolrXmlPersistence, TestReversedWildcardFilterFactory, TestLFUCache, DocValuesTest, FileBasedSpellCheckerTest, DirectSolrConnectionTest, TestDynamicFieldResource, TestMaxScoreQueryParser, TestReplicationHandler, TestPHPSerializedResponseWriter, TestDynamicFieldCollectionResource, TestFunctionQuery, ShardRoutingTest, PreAnalyzedFieldTest, SolrCoreTest, TestDefaultSearchFieldResource, ClusterStateUpdateTest, TestCopyFieldCollectionResource, MBeansHandlerTest, CachingDirectoryFactoryTest, BasicFunctionalityTest, TestSolrDeletionPolicy1, SyncSliceTest, JsonLoaderTest, TermVectorComponentTest, TestAddFieldRealTimeGet, OverseerTest, TestSchemaVersionResource, TestRealTimeGet, ShowFileRequestHandlerTest, TestSolrDeletionPolicy2, UUIDFieldTest, BadComponentTest, CircularListTest, TestSolrQueryParser, SuggesterTest, SliceStateTest, BasicDistributedZkTest]
> [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=BasicDistributedZkTest -Dtests.seed=33D44819CFDF4DF5 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar_BH -Dtests.timezone=Europe/Helsinki -Dtests.file.encoding=ISO-8859-1
> [junit4] ERROR 0.00s J0 | BasicDistributedZkTest (suite) <<<
> [junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest:
> [junit4] > 1) Thread[id=3239, name=recoveryCmdExecutor-1407-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
> [junit4] > at java.net.PlainSocketImpl.socketConnect(Native Method)
> [junit4] > at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
> [junit4] > at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
> [junit4] > at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
> [junit4] > at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391)
> [junit4] > at java.net.Socket.connect(Socket.java:579)
> [junit4] > at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
> [junit4] > at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
> [junit4] > at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
> [junit4] > at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
> [junit4] > at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
> [junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
> [junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
> [junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
> [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
> [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
> [junit4] > at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291)
> [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> [junit4] > at java.lang.Thread.run(Thread.java:722)
> [junit4] > at __randomizedtesting.SeedInfo.seed([33D44819CFDF4DF5]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
> [junit4] > 1) Thread[id=3239, name=recoveryCmdExecutor-1407-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
> [junit4] > at java.net.PlainSocketImpl.socketConnect(Native Method)
> [junit4] > at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
> [junit4] > at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
> [junit4] > at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
> [junit4] > at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391)
> [junit4] > at java.net.Socket.connect(Socket.java:579)
> [junit4] > at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
> [junit4] > at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
> [junit4] > at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
> [junit4] > at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
> [junit4] > at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
> [junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
> [junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
> [junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
> [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
> [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
> [junit4] > at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291)
> [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> [junit4] > at java.lang.Thread.run(Thread.java:722)
> [junit4] > at __randomizedtesting.SeedInfo.seed([33D44819CFDF4DF5]:0)
> [junit4] Completed on J0 in 394.33s, 1 test, 2 errors <<< FAILURES!
>
> [...truncated 519 lines...]
> BUILD FAILED
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/build.xml:389: The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/build.xml:369: The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/build.xml:39: The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build.xml:181: The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/common-build.xml:449: The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/lucene/common-build.xml:1247: The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/lucene/common-build.xml:890: There were test failures: 318 suites, 1342 tests, 2 suite-level errors, 642 ignored (6 assumptions)
>
> Total time: 42 minutes 53 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
[JENKINS] Lucene-Solr-Tests-trunk-Java7 - Build # 4164 - Still
Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-trunk-Java7/4164/
2 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.BasicDistributedZkTest
Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest: 1) Thread[id=3239, name=recoveryCmdExecutor-1407-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at java.net.PlainSocketImpl.socketConnect(Native Method) at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339) at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200) at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182) at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391) at java.net.Socket.connect(Socket.java:579) at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127) at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180) at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294) at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645) at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180) at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:722)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest:
1) Thread[id=3239, name=recoveryCmdExecutor-1407-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391)
at java.net.Socket.connect(Socket.java:579)
at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:722)
at __randomizedtesting.SeedInfo.seed([33D44819CFDF4DF5]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.BasicDistributedZkTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=3239, name=recoveryCmdExecutor-1407-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest] at java.net.PlainSocketImpl.socketConnect(Native Method) at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339) at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200) at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182) at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391) at java.net.Socket.connect(Socket.java:579) at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127) at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180) at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294) at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645) at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180) at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:722)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=3239, name=recoveryCmdExecutor-1407-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391)
at java.net.Socket.connect(Socket.java:579)
at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:722)
at __randomizedtesting.SeedInfo.seed([33D44819CFDF4DF5]:0)
Build Log:
[...truncated 9682 lines...]
[junit4] Suite: org.apache.solr.cloud.BasicDistributedZkTest
[junit4] 2> 838345 T2789 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /rwd/ns
[junit4] 2> 838352 T2789 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4] 2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZkTest-1374495821781
[junit4] 2> 838353 T2789 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 838354 T2790 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 838455 T2789 oasc.ZkTestServer.run start zk server on port:57977
[junit4] 2> 838456 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 838460 T2796 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@20a5e53a name:ZooKeeperConnection Watcher:127.0.0.1:57977 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 838460 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 838461 T2789 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 838464 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 838473 T2798 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45dbee8d name:ZooKeeperConnection Watcher:127.0.0.1:57977/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 838474 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 838474 T2789 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4] 2> 838478 T2789 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4] 2> 838487 T2789 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4] 2> 838489 T2789 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4] 2> 838491 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 838492 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4] 2> 838502 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 838503 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4] 2> 838606 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 838608 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4] 2> 838610 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 838611 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4] 2> 838638 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 838639 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4] 2> 838642 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 838643 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4] 2> 838684 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 838685 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 838727 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 838728 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4] 2> 838731 T2789 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 838731 T2789 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4] 2> 839069 T2789 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 839075 T2789 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:57982
[junit4] 2> 839075 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 839076 T2789 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 839077 T2789 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225
[junit4] 2> 839077 T2789 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/'
[junit4] 2> 839115 T2789 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/solr.xml
[junit4] 2> 839206 T2789 oasc.CoreContainer.<init> New CoreContainer 454265048
[junit4] 2> 839207 T2789 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/]
[junit4] 2> 839209 T2789 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4] 2> 839209 T2789 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 839210 T2789 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 839210 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 839211 T2789 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 839211 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 839212 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 839212 T2789 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 839213 T2789 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 839214 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4] 2> 839230 T2789 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 839231 T2789 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57977/solr
[junit4] 2> 839231 T2789 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 839232 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 839239 T2809 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a5da037 name:ZooKeeperConnection Watcher:127.0.0.1:57977 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 839239 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 839241 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4] 2> 839252 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 839257 T2811 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6b45e390 name:ZooKeeperConnection Watcher:127.0.0.1:57977/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 839257 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 839259 T2789 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 839270 T2789 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 839273 T2789 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 839275 T2789 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57982_rwd%2Fns
[junit4] 2> 839283 T2789 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57982_rwd%2Fns
[junit4] 2> 839286 T2789 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 839301 T2789 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 839309 T2789 oasc.Overseer.start Overseer (id=90078958176763907-127.0.0.1:57982_rwd%2Fns-n_0000000000) starting
[junit4] 2> 839323 T2789 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 839326 T2813 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 839327 T2789 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 839329 T2789 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 839331 T2789 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 839333 T2812 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 839338 T2814 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 839338 T2814 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 839340 T2814 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 840837 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 840838 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "numShards":"1",
[junit4] 2> "shard":null,
[junit4] 2> "roles":null,
[junit4] 2> "state":"down",
[junit4] 2> "shard_state":"active",
[junit4] 2> "core":"collection1",
[junit4] 2> "collection":"control_collection",
[junit4] 2> "shard_range":null,
[junit4] 2> "node_name":"127.0.0.1:57982_rwd%2Fns",
[junit4] 2> "base_url":"http://127.0.0.1:57982/rwd/ns"}
[junit4] 2> 840838 T2812 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
[junit4] 2> 840839 T2812 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 840855 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 841342 T2814 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
[junit4] 2> 841342 T2814 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/collection1
[junit4] 2> 841342 T2814 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4] 2> 841343 T2814 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4] 2> 841343 T2814 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4] 2> 841344 T2814 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/collection1/'
[junit4] 2> 841346 T2814 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/collection1/lib/README' to classloader
[junit4] 2> 841347 T2814 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/collection1/lib/classes/' to classloader
[junit4] 2> 841406 T2814 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4] 2> 841470 T2814 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4] 2> 841571 T2814 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4] 2> 841583 T2814 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4] 2> 842242 T2814 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4] 2> 842256 T2814 oass.IndexSchema.readSchema default search field in schema is text
[junit4] 2> 842260 T2814 oass.IndexSchema.readSchema unique key field: id
[junit4] 2> 842273 T2814 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 842278 T2814 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 842283 T2814 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 842284 T2814 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 842284 T2814 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 842285 T2814 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 842286 T2814 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 842286 T2814 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 842286 T2814 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4] 2> 842287 T2814 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1374495822225/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/control/data/
[junit4] 2> 842287 T2814 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5de17399
[junit4] 2> 842288 T2814 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/control/data
[junit4] 2> 842289 T2814 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/control/data/index/
[junit4] 2> 842289 T2814 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/control/data/index' doesn't exist. Creating new index...
[junit4] 2> 842289 T2814 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/control/data/index
[junit4] 2> 842294 T2814 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@203e9b48 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7e3b398a),segFN=segments_1,generation=1}
[junit4] 2> 842295 T2814 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 842300 T2814 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4] 2> 842300 T2814 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4] 2> 842301 T2814 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 842302 T2814 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4] 2> 842303 T2814 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4] 2> 842303 T2814 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4] 2> 842303 T2814 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4] 2> 842304 T2814 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4] 2> 842304 T2814 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4] 2> 842321 T2814 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4] 2> 842329 T2814 oass.SolrIndexSearcher.<init> Opening Searcher@52ea3d41 main
[junit4] 2> 842330 T2814 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4] 2> 842331 T2814 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4] 2> 842334 T2815 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@52ea3d41 main{StandardDirectoryReader(segments_1:1)}
[junit4] 2> 842337 T2814 oasc.CoreContainer.registerCore registering core: collection1
[junit4] 2> 842337 T2814 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57982/rwd/ns collection:control_collection shard:shard1
[junit4] 2> 842337 T2814 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
[junit4] 2> 842356 T2814 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4] 2> 842377 T2814 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
[junit4] 2> 842388 T2814 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4] 2> 842388 T2814 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4] 2> 842388 T2814 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57982/rwd/ns/collection1/
[junit4] 2> 842389 T2814 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4] 2> 842389 T2814 oasc.SyncStrategy.syncToMe http://127.0.0.1:57982/rwd/ns/collection1/ has no replicas
[junit4] 2> 842389 T2814 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57982/rwd/ns/collection1/ shard1
[junit4] 2> 842390 T2814 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4] 2> 843864 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 843888 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 843925 T2814 oasc.ZkController.register We are http://127.0.0.1:57982/rwd/ns/collection1/ and leader is http://127.0.0.1:57982/rwd/ns/collection1/
[junit4] 2> 843925 T2814 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57982/rwd/ns
[junit4] 2> 843925 T2814 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4] 2> 843926 T2814 oasc.ZkController.publish publishing core=collection1 state=active
[junit4] 2> 843926 T2814 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 843928 T2814 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 843929 T2789 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0
[junit4] 2> 843930 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4] 2> 843930 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4] 2> 843943 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:
[junit4] 2> 843944 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 843947 T2818 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e187f0c name:ZooKeeperConnection Watcher:127.0.0.1:57977/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 843947 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 843948 T2789 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 843950 T2789 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
[junit4] 2> 844271 T2789 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 844275 T2789 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:57986
[junit4] 2> 844275 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 844276 T2789 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 844277 T2789 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378
[junit4] 2> 844277 T2789 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/'
[junit4] 2> 844317 T2789 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/solr.xml
[junit4] 2> 844409 T2789 oasc.CoreContainer.<init> New CoreContainer 1420732659
[junit4] 2> 844410 T2789 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/]
[junit4] 2> 844412 T2789 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4] 2> 844413 T2789 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 844413 T2789 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 844414 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 844414 T2789 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 844415 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 844415 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 844416 T2789 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 844417 T2789 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 844417 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4] 2> 844433 T2789 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 844434 T2789 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57977/solr
[junit4] 2> 844434 T2789 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 844436 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 844444 T2829 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4741fcc1 name:ZooKeeperConnection Watcher:127.0.0.1:57977 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 844444 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 844447 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4] 2> 844458 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 844460 T2831 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c0ab1f9 name:ZooKeeperConnection Watcher:127.0.0.1:57977/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 844461 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 844474 T2789 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 845392 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 845393 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "numShards":"1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "roles":null,
[junit4] 2> "state":"active",
[junit4] 2> "shard_state":"active",
[junit4] 2> "core":"collection1",
[junit4] 2> "collection":"control_collection",
[junit4] 2> "shard_range":null,
[junit4] 2> "node_name":"127.0.0.1:57982_rwd%2Fns",
[junit4] 2> "base_url":"http://127.0.0.1:57982/rwd/ns"}
[junit4] 2> 845436 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 845436 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 845436 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 845478 T2789 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57986_rwd%2Fns
[junit4] 2> 845480 T2789 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57986_rwd%2Fns
[junit4] 2> 845482 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4] 2> 845482 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4] 2> 845482 T2831 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 845483 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4] 2> 845483 T2818 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 845491 T2811 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 845498 T2832 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 845499 T2832 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 845500 T2832 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 846940 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 846942 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "numShards":"2",
[junit4] 2> "shard":null,
[junit4] 2> "roles":null,
[junit4] 2> "state":"down",
[junit4] 2> "shard_state":"active",
[junit4] 2> "core":"collection1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard_range":null,
[junit4] 2> "node_name":"127.0.0.1:57986_rwd%2Fns",
[junit4] 2> "base_url":"http://127.0.0.1:57986/rwd/ns"}
[junit4] 2> 846942 T2812 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 2
[junit4] 2> 846942 T2812 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 846959 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 846959 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 846959 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 847502 T2832 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
[junit4] 2> 847502 T2832 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/collection1
[junit4] 2> 847502 T2832 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4] 2> 847503 T2832 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4] 2> 847503 T2832 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4] 2> 847504 T2832 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/collection1/'
[junit4] 2> 847506 T2832 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/collection1/lib/classes/' to classloader
[junit4] 2> 847507 T2832 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/collection1/lib/README' to classloader
[junit4] 2> 847559 T2832 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4] 2> 847604 T2832 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4] 2> 847706 T2832 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4] 2> 847715 T2832 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4] 2> 848185 T2832 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4] 2> 848195 T2832 oass.IndexSchema.readSchema default search field in schema is text
[junit4] 2> 848198 T2832 oass.IndexSchema.readSchema unique key field: id
[junit4] 2> 848229 T2832 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 848233 T2832 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 848237 T2832 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 848237 T2832 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 848237 T2832 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 848238 T2832 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 848238 T2832 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 848239 T2832 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 848239 T2832 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4] 2> 848239 T2832 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1374495827378/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty1/
[junit4] 2> 848239 T2832 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5de17399
[junit4] 2> 848240 T2832 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty1
[junit4] 2> 848240 T2832 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty1/index/
[junit4] 2> 848241 T2832 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty1/index' doesn't exist. Creating new index...
[junit4] 2> 848241 T2832 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty1/index
[junit4] 2> 848245 T2832 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@68ad8c59 lockFactory=org.apache.lucene.store.NativeFSLockFactory@581f222d),segFN=segments_1,generation=1}
[junit4] 2> 848245 T2832 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 848249 T2832 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4] 2> 848250 T2832 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4] 2> 848250 T2832 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 848251 T2832 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4] 2> 848251 T2832 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4] 2> 848252 T2832 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4] 2> 848252 T2832 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4] 2> 848252 T2832 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4] 2> 848253 T2832 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4] 2> 848265 T2832 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4] 2> 848271 T2832 oass.SolrIndexSearcher.<init> Opening Searcher@7e54a378 main
[junit4] 2> 848272 T2832 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4] 2> 848272 T2832 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4] 2> 848276 T2833 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7e54a378 main{StandardDirectoryReader(segments_1:1)}
[junit4] 2> 848277 T2832 oasc.CoreContainer.registerCore registering core: collection1
[junit4] 2> 848277 T2832 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57986/rwd/ns collection:collection1 shard:shard1
[junit4] 2> 848278 T2832 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
[junit4] 2> 848293 T2832 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 848315 T2832 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
[junit4] 2> 848326 T2832 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4] 2> 848326 T2832 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4] 2> 848326 T2832 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57986/rwd/ns/collection1/
[junit4] 2> 848327 T2832 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4] 2> 848327 T2832 oasc.SyncStrategy.syncToMe http://127.0.0.1:57986/rwd/ns/collection1/ has no replicas
[junit4] 2> 848327 T2832 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57986/rwd/ns/collection1/ shard1
[junit4] 2> 848328 T2832 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 848463 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 848480 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 848480 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 848480 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 848487 T2832 oasc.ZkController.register We are http://127.0.0.1:57986/rwd/ns/collection1/ and leader is http://127.0.0.1:57986/rwd/ns/collection1/
[junit4] 2> 848487 T2832 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57986/rwd/ns
[junit4] 2> 848487 T2832 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4] 2> 848487 T2832 oasc.ZkController.publish publishing core=collection1 state=active
[junit4] 2> 848488 T2832 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 848498 T2832 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 848499 T2789 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0
[junit4] 2> 848500 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4] 2> 848500 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4] 2> 848808 T2789 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 848812 T2789 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:57989
[junit4] 2> 848812 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 848813 T2789 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 848813 T2789 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937
[junit4] 2> 848814 T2789 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/'
[junit4] 2> 848856 T2789 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/solr.xml
[junit4] 2> 848958 T2789 oasc.CoreContainer.<init> New CoreContainer 1108024381
[junit4] 2> 848959 T2789 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/]
[junit4] 2> 848961 T2789 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4] 2> 848962 T2789 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 848962 T2789 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 848963 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 848964 T2789 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 848964 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 848965 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 848965 T2789 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 848966 T2789 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 848966 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4] 2> 848986 T2789 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 848988 T2789 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57977/solr
[junit4] 2> 848988 T2789 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 848989 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 849032 T2845 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@33e605f4 name:ZooKeeperConnection Watcher:127.0.0.1:57977 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 849033 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 849067 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4] 2> 849081 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 849084 T2847 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2119f463 name:ZooKeeperConnection Watcher:127.0.0.1:57977/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 849084 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 849138 T2789 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 849986 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 849988 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "numShards":"2",
[junit4] 2> "shard":"shard1",
[junit4] 2> "roles":null,
[junit4] 2> "state":"active",
[junit4] 2> "shard_state":"active",
[junit4] 2> "core":"collection1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard_range":null,
[junit4] 2> "node_name":"127.0.0.1:57986_rwd%2Fns",
[junit4] 2> "base_url":"http://127.0.0.1:57986/rwd/ns"}
[junit4] 2> 850020 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 850020 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 850020 T2847 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 850020 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4] 2> 850142 T2789 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57989_rwd%2Fns
[junit4] 2> 850171 T2789 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57989_rwd%2Fns
[junit4] 2> 850173 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4] 2> 850173 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4] 2> 850174 T2847 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 850174 T2831 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 850174 T2847 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4] 2> 850174 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4] 2> 850175 T2818 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 850188 T2811 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 850195 T2848 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 850195 T2848 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 850197 T2848 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 851532 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 851533 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "core_node_name":"core_node2",
[junit4] 2> "numShards":"2",
[junit4] 2> "shard":null,
[junit4] 2> "roles":null,
[junit4] 2> "state":"down",
[junit4] 2> "shard_state":"active",
[junit4] 2> "core":"collection1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard_range":null,
[junit4] 2> "node_name":"127.0.0.1:57989_rwd%2Fns",
[junit4] 2> "base_url":"http://127.0.0.1:57989/rwd/ns"}
[junit4] 2> 851534 T2812 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4] 2> 851534 T2812 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
[junit4] 2> 851551 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 851551 T2847 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 851551 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 851551 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 852199 T2848 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
[junit4] 2> 852199 T2848 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/collection1
[junit4] 2> 852200 T2848 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4] 2> 852200 T2848 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4] 2> 852201 T2848 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4] 2> 852201 T2848 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/collection1/'
[junit4] 2> 852203 T2848 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/collection1/lib/README' to classloader
[junit4] 2> 852204 T2848 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/collection1/lib/classes/' to classloader
[junit4] 2> 852262 T2848 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4] 2> 852332 T2848 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4] 2> 852434 T2848 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4] 2> 852447 T2848 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4] 2> 853099 T2848 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4] 2> 853109 T2848 oass.IndexSchema.readSchema default search field in schema is text
[junit4] 2> 853112 T2848 oass.IndexSchema.readSchema unique key field: id
[junit4] 2> 853121 T2848 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 853125 T2848 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 853128 T2848 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 853129 T2848 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 853129 T2848 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 853129 T2848 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 853130 T2848 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 853130 T2848 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 853130 T2848 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4] 2> 853131 T2848 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1374495831937/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty2/
[junit4] 2> 853131 T2848 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5de17399
[junit4] 2> 853132 T2848 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty2
[junit4] 2> 853132 T2848 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty2/index/
[junit4] 2> 853132 T2848 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty2/index' doesn't exist. Creating new index...
[junit4] 2> 853132 T2848 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty2/index
[junit4] 2> 853135 T2848 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5611756f lockFactory=org.apache.lucene.store.NativeFSLockFactory@314b85a),segFN=segments_1,generation=1}
[junit4] 2> 853136 T2848 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 853140 T2848 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4] 2> 853140 T2848 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4] 2> 853141 T2848 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 853141 T2848 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4] 2> 853142 T2848 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4] 2> 853142 T2848 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4] 2> 853142 T2848 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4] 2> 853143 T2848 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4] 2> 853143 T2848 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4] 2> 853156 T2848 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4] 2> 853163 T2848 oass.SolrIndexSearcher.<init> Opening Searcher@6fcd2a55 main
[junit4] 2> 853164 T2848 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4] 2> 853164 T2848 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4] 2> 853167 T2849 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6fcd2a55 main{StandardDirectoryReader(segments_1:1)}
[junit4] 2> 853169 T2848 oasc.CoreContainer.registerCore registering core: collection1
[junit4] 2> 853169 T2848 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57989/rwd/ns collection:collection1 shard:shard2
[junit4] 2> 853169 T2848 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
[junit4] 2> 853180 T2848 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4] 2> 853196 T2848 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
[junit4] 2> 853204 T2848 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4] 2> 853204 T2848 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4] 2> 853204 T2848 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57989/rwd/ns/collection1/
[junit4] 2> 853204 T2848 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4] 2> 853205 T2848 oasc.SyncStrategy.syncToMe http://127.0.0.1:57989/rwd/ns/collection1/ has no replicas
[junit4] 2> 853205 T2848 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57989/rwd/ns/collection1/ shard2
[junit4] 2> 853205 T2848 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4] 2> 854558 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 854601 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 854601 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 854601 T2847 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 854601 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 854645 T2848 oasc.ZkController.register We are http://127.0.0.1:57989/rwd/ns/collection1/ and leader is http://127.0.0.1:57989/rwd/ns/collection1/
[junit4] 2> 854645 T2848 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57989/rwd/ns
[junit4] 2> 854645 T2848 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4] 2> 854646 T2848 oasc.ZkController.publish publishing core=collection1 state=active
[junit4] 2> 854646 T2848 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 854648 T2848 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 854649 T2789 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0
[junit4] 2> 854650 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4] 2> 854650 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4] 2> 854970 T2789 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4] 2> 854974 T2789 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:57992
[junit4] 2> 854975 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4] 2> 854976 T2789 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 854976 T2789 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091
[junit4] 2> 854977 T2789 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/'
[junit4] 2> 855022 T2789 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/solr.xml
[junit4] 2> 855122 T2789 oasc.CoreContainer.<init> New CoreContainer 1041480000
[junit4] 2> 855123 T2789 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/]
[junit4] 2> 855125 T2789 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4] 2> 855126 T2789 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4] 2> 855126 T2789 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4] 2> 855127 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4] 2> 855127 T2789 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4] 2> 855128 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4] 2> 855128 T2789 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4] 2> 855129 T2789 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4] 2> 855129 T2789 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4] 2> 855130 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4] 2> 855147 T2789 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 855148 T2789 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57977/solr
[junit4] 2> 855148 T2789 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4] 2> 855149 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 855152 T2861 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7ad6e636 name:ZooKeeperConnection Watcher:127.0.0.1:57977 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 855153 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 855166 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4] 2> 855176 T2789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 855178 T2863 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@55a547bc name:ZooKeeperConnection Watcher:127.0.0.1:57977/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 855179 T2789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 855190 T2789 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 856115 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 856116 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "core_node_name":"core_node2",
[junit4] 2> "numShards":"2",
[junit4] 2> "shard":"shard2",
[junit4] 2> "roles":null,
[junit4] 2> "state":"active",
[junit4] 2> "shard_state":"active",
[junit4] 2> "core":"collection1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard_range":null,
[junit4] 2> "node_name":"127.0.0.1:57989_rwd%2Fns",
[junit4] 2> "base_url":"http://127.0.0.1:57989/rwd/ns"}
[junit4] 2> 856152 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 856152 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 856152 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 856152 T2863 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 856152 T2847 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4] 2> 856193 T2789 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57992_rwd%2Fns
[junit4] 2> 856195 T2789 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57992_rwd%2Fns
[junit4] 2> 856197 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4] 2> 856197 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4] 2> 856198 T2847 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 856198 T2863 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 856198 T2831 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 856199 T2863 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4] 2> 856198 T2847 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4] 2> 856199 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4] 2> 856207 T2818 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 856207 T2811 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 856214 T2864 oasc.ZkController.publish publishing core=collection1 state=down
[junit4] 2> 856214 T2864 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 856216 T2864 oasc.ZkController.waitForCoreNodeName look for our core node name
[junit4] 2> 857656 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 857657 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "core_node_name":"core_node3",
[junit4] 2> "numShards":"2",
[junit4] 2> "shard":null,
[junit4] 2> "roles":null,
[junit4] 2> "state":"down",
[junit4] 2> "shard_state":"active",
[junit4] 2> "core":"collection1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "shard_range":null,
[junit4] 2> "node_name":"127.0.0.1:57992_rwd%2Fns",
[junit4] 2> "base_url":"http://127.0.0.1:57992/rwd/ns"}
[junit4] 2> 857658 T2812 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4] 2> 857658 T2812 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 857707 T2863 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4] 2> 857707 T2831 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4] 2> 857707 T2847 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4] 2> 857707 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4] 2> 857707 T2811 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4] 2> 858218 T2864 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
[junit4] 2> 858218 T2864 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/collection1
[junit4] 2> 858218 T2864 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4] 2> 858219 T2864 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4] 2> 858219 T2864 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4] 2> 858220 T2864 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/collection1/'
[junit4] 2> 858222 T2864 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/collection1/lib/README' to classloader
[junit4] 2> 858222 T2864 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/collection1/lib/classes/' to classloader
[junit4] 2> 858291 T2864 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
[junit4] 2> 858338 T2864 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4] 2> 858440 T2864 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4] 2> 858449 T2864 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4] 2> 859063 T2864 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4] 2> 859074 T2864 oass.IndexSchema.readSchema default search field in schema is text
[junit4] 2> 859077 T2864 oass.IndexSchema.readSchema unique key field: id
[junit4] 2> 859088 T2864 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 859092 T2864 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4] 2> 859096 T2864 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 859097 T2864 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 859097 T2864 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 859097 T2864 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 859098 T2864 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4] 2> 859099 T2864 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4] 2> 859099 T2864 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4] 2> 859099 T2864 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1374495838091/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty3/
[junit4] 2> 859099 T2864 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5de17399
[junit4] 2> 859101 T2864 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty3
[junit4] 2> 859101 T2864 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty3/index/
[junit4] 2> 859102 T2864 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty3/index' doesn't exist. Creating new index...
[junit4] 2> 859102 T2864 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1374495821780/jetty3/index
[junit4] 2> 859105 T2864 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@24446f84 lockFactory=org.apache.lucene.store.NativeFSLockFactory@bb29558),segFN=segments_1,generation=1}
[junit4] 2> 859106 T2864 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4] 2> 859110 T2864 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4] 2> 859110 T2864 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4] 2> 859111 T2864 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 859112 T2864 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4] 2> 859112 T2864 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4] 2> 859112 T2864 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4] 2> 859113 T2864 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4] 2> 859113 T2864 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4] 2> 859113 T2864 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4] 2> 859128 T2864 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4] 2> 859136 T2864 oass.SolrIndexSearcher.<init> Opening Searcher@67e5d2fa main
[junit4] 2> 859137 T2864 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4] 2> 859137 T2864 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4] 2> 859140 T2865 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@67e5d2fa main{StandardDirectoryReader(segments_1:1)}
[junit4] 2> 859142 T2864 oasc.CoreContainer.registerCore registering core: collection1
[junit4] 2> 859142 T2864 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57992/rwd/ns collection:collection1 shard:shard1
[junit4] 2> 859142 T2864 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
[junit4] 2> 859165 T2864 oasc.ZkController.register We are http://127.0.0.1:57992/rwd/ns/collection1/ and leader is http://127.0.0.1:57986/rwd/ns/collection1/
[junit4] 2> 859165 T2864 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57992/rwd/ns
[junit4] 2> 859165 T2864 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4] 2> 859165 T2864 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4] 2> ASYNC NEW_CORE C624 name=collection1 org.apache.solr.core.SolrCore@b98d041 url=http://127.0.0.1:57992/rwd/ns/collection1 node=127.0.0.1:57992_rwd%2Fns C624_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, node_name=127.0.0.1:57992_rwd%2Fns, base_url=http://127.0.0.1:57992/rwd/ns}
[junit4] 2> 859166 T2866 C624 P57992 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4] 2> 859166 T2864 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 859166 T2866 C624 P57992 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4] 2> 859167 T2866 C624 P57992 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4] 2> 859167 T2866 C624 P57992 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4] 2> 859167 T2789 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J0
[junit4] 2> 859168 T2789 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4] 2> 859169 T2789 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4] 2> 859173 T2866 C624 P57992 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4] 2> 859185 T2821 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true
[junit4] 2> 859211 T2812 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 859211 T2812 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4] 2> "operation":"state",
[junit4] 2> "core_node_name":"core_node3",
[junit4] 2> "numShards":"2",
[junit4] 2> "shard":"shard1",
[junit4] 2> "roles":null,
[junit4] 2> "state":"recovering",
[junit4] 2> "shard_state":"active",
[junit4] 2> "core":"collection1",
[ju
[...truncated too long message...]
ry doPackFST= true), range_facet_sl=PostingsFormat(name=Memory doPackFST= true), range_facet_si=PostingsFormat(name=MockRandom), oddField_s=PostingsFormat(name=Memory doPackFST= true), sequence_i=PostingsFormat(name=Memory doPackFST= true), name=PostingsFormat(name=Memory doPackFST= true), foo_i=PostingsFormat(name=TestBloomFilteredLucene41Postings), regex_dup_B_s=PostingsFormat(name=MockRandom), multiDefault=PostingsFormat(name=MockRandom), n_tf1=PostingsFormat(name=MockRandom), n_dt1=PostingsFormat(name=TestBloomFilteredLucene41Postings), genre_s=PostingsFormat(name=Memory doPackFST= true), author_t=PostingsFormat(name=TestBloomFilteredLucene41Postings), range_facet_l=PostingsFormat(name=MockRandom), n_ti1=PostingsFormat(name=Memory doPackFST= true), text=Pulsing41(freqCutoff=4 minBlockSize=87 maxBlockSize=218), _version_=PostingsFormat(name=TestBloomFilteredLucene41Postings), val_i=PostingsFormat(name=Memory doPackFST= true), SubjectTerms_mfacet=PostingsFormat(name=TestBloomFilteredLucene41Postings), series_t=PostingsFormat(name=TestBloomFilteredLucene41Postings), a_t=PostingsFormat(name=MockRandom), n_tdt1=PostingsFormat(name=TestBloomFilteredLucene41Postings), regex_dup_A_s=Pulsing41(freqCutoff=4 minBlockSize=87 maxBlockSize=218), price=Pulsing41(freqCutoff=4 minBlockSize=87 maxBlockSize=218), other_tl1=PostingsFormat(name=TestBloomFilteredLucene41Postings), n_l1=PostingsFormat(name=MockRandom), a_si=PostingsFormat(name=TestBloomFilteredLucene41Postings), inStock=Pulsing41(freqCutoff=4 minBlockSize=87 maxBlockSize=218)}, docValues:{timestamp=DocValuesFormat(name=Lucene42)}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=ar_BH, timezone=Europe/Helsinki
[junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_21 (64-bit)/cpus=16,threads=2,free=238764256,total=399507456
[junit4] 2> NOTE: All tests run in this JVM: [TestPerFieldSimilarity, TestQueryUtils, TestSurroundQueryParser, QueryEqualityTest, PeerSyncTest, SolrTestCaseJ4Test, TestPropInjectDefaults, RecoveryZkTest, TestStressRecovery, UpdateRequestProcessorFactoryTest, QueryElevationComponentTest, TestSolrXml, StatelessScriptUpdateProcessorFactoryTest, ScriptEngineTest, TestSchemaNameResource, TestSerializedLuceneMatchVersion, TestRandomDVFaceting, PolyFieldTest, RegexBoostProcessorTest, AutoCommitTest, TestFastLRUCache, LegacyHTMLStripCharFilterTest, ZkCLITest, AddSchemaFieldsUpdateProcessorFactoryTest, DateFieldTest, AssignTest, SOLR749Test, TermsComponentTest, TestSolrXmlPersistence, TestReversedWildcardFilterFactory, TestLFUCache, DocValuesTest, FileBasedSpellCheckerTest, DirectSolrConnectionTest, TestDynamicFieldResource, TestMaxScoreQueryParser, TestReplicationHandler, TestPHPSerializedResponseWriter, TestDynamicFieldCollectionResource, TestFunctionQuery, ShardRoutingTest, PreAnalyzedFieldTest, SolrCoreTest, TestDefaultSearchFieldResource, ClusterStateUpdateTest, TestCopyFieldCollectionResource, MBeansHandlerTest, CachingDirectoryFactoryTest, BasicFunctionalityTest, TestSolrDeletionPolicy1, SyncSliceTest, JsonLoaderTest, TermVectorComponentTest, TestAddFieldRealTimeGet, OverseerTest, TestSchemaVersionResource, TestRealTimeGet, ShowFileRequestHandlerTest, TestSolrDeletionPolicy2, UUIDFieldTest, BadComponentTest, CircularListTest, TestSolrQueryParser, SuggesterTest, SliceStateTest, BasicDistributedZkTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=BasicDistributedZkTest -Dtests.seed=33D44819CFDF4DF5 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar_BH -Dtests.timezone=Europe/Helsinki -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 0.00s J0 | BasicDistributedZkTest (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest:
[junit4] > 1) Thread[id=3239, name=recoveryCmdExecutor-1407-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
[junit4] > at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] > at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
[junit4] > at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
[junit4] > at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
[junit4] > at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391)
[junit4] > at java.net.Socket.connect(Socket.java:579)
[junit4] > at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
[junit4] > at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
[junit4] > at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
[junit4] > at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
[junit4] > at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
[junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
[junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
[junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
[junit4] > at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[junit4] > at java.lang.Thread.run(Thread.java:722)
[junit4] > at __randomizedtesting.SeedInfo.seed([33D44819CFDF4DF5]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=3239, name=recoveryCmdExecutor-1407-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
[junit4] > at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] > at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
[junit4] > at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
[junit4] > at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
[junit4] > at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:391)
[junit4] > at java.net.Socket.connect(Socket.java:579)
[junit4] > at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
[junit4] > at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
[junit4] > at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
[junit4] > at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
[junit4] > at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
[junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
[junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
[junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
[junit4] > at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:291)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[junit4] > at java.lang.Thread.run(Thread.java:722)
[junit4] > at __randomizedtesting.SeedInfo.seed([33D44819CFDF4DF5]:0)
[junit4] Completed on J0 in 394.33s, 1 test, 2 errors <<< FAILURES!
[...truncated 519 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/build.xml:389: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/build.xml:369: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build.xml:181: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/common-build.xml:449: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/lucene/common-build.xml:1247: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/lucene/common-build.xml:890: There were test failures: 318 suites, 1342 tests, 2 suite-level errors, 642 ignored (6 assumptions)
Total time: 42 minutes 53 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure