You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/01/09 18:11:49 UTC
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3767
- Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3767/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC
1 tests failed.
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
timeout waiting to see all nodes active
Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
at __randomizedtesting.SeedInfo.seed([8A0AC4DB45F62808:25EFB01EB0A45F0]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:311)
at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:262)
at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:244)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12372 lines...]
[junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/init-core-data-001
[junit4] 2> 2502369 INFO (SUITE-PeerSyncReplicationTest-seed#[8A0AC4DB45F62808]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 2502404 INFO (SUITE-PeerSyncReplicationTest-seed#[8A0AC4DB45F62808]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 2502407 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2502408 INFO (Thread-3889) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2502408 INFO (Thread-3889) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2502518 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.ZkTestServer start zk server on port:60356
[junit4] 2> 2502551 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2502597 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 2502697 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2502717 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2502744 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2502776 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2502799 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2502817 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2502836 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2502842 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2502859 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2507484 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/control-001/cores/collection1
[junit4] 2> 2507496 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2507497 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5ae37f4b{/,null,AVAILABLE}
[junit4] 2> 2507501 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@52727f84{SSL,[ssl, http/1.1]}{127.0.0.1:60366}
[junit4] 2> 2507501 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.Server Started @2513049ms
[junit4] 2> 2507501 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/tempDir-001/control/data, hostContext=/, hostPort=60366, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/control-001/cores}
[junit4] 2> 2507501 ERROR (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2507502 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2507502 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2507502 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2507502 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-09T17:27:37.455Z
[junit4] 2> 2507507 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2507507 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/control-001/solr.xml
[junit4] 2> 2507527 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60356/solr
[junit4] 2> 2507578 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60366_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60366_
[junit4] 2> 2507580 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60366_ ] o.a.s.c.Overseer Overseer (id=97254300212461573-127.0.0.1:60366_-n_0000000000) starting
[junit4] 2> 2507598 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60366_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60366_
[junit4] 2> 2507602 INFO (zkCallback-2754-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2507670 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60366_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/control-001/cores
[junit4] 2> 2507670 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60366_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2507676 INFO (OverseerStateUpdate-97254300212461573-127.0.0.1:60366_-n_0000000000) [n:127.0.0.1:60366_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2508699 WARN (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2508700 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2508713 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2508813 WARN (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2508816 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2508836 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 2508836 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/control-001/cores/collection1/data/]
[junit4] 2> 2508836 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@24b1deb
[junit4] 2> 2508839 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=84.708984375, floorSegmentMB=0.6923828125, forceMergeDeletesPctAllowed=13.691854934993254, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2508851 WARN (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2508864 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2508864 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2508865 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2508865 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2508866 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=5.439453125, floorSegmentMB=0.2607421875, forceMergeDeletesPctAllowed=11.722461350720005, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 2508867 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3340b5d4[collection1] main]
[junit4] 2> 2508868 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2508869 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2508869 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2508870 INFO (searcherExecutor-8786-thread-1-processing-n:127.0.0.1:60366_ x:collection1 c:control_collection) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3340b5d4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2508871 INFO (coreLoadExecutor-8785-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556068810174234624
[junit4] 2> 2508881 INFO (coreZkRegister-8778-thread-1-processing-n:127.0.0.1:60366_ x:collection1 c:control_collection) [n:127.0.0.1:60366_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2508881 INFO (coreZkRegister-8778-thread-1-processing-n:127.0.0.1:60366_ x:collection1 c:control_collection) [n:127.0.0.1:60366_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2508881 INFO (coreZkRegister-8778-thread-1-processing-n:127.0.0.1:60366_ x:collection1 c:control_collection) [n:127.0.0.1:60366_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:60366/collection1/
[junit4] 2> 2508881 INFO (coreZkRegister-8778-thread-1-processing-n:127.0.0.1:60366_ x:collection1 c:control_collection) [n:127.0.0.1:60366_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2508881 INFO (coreZkRegister-8778-thread-1-processing-n:127.0.0.1:60366_ x:collection1 c:control_collection) [n:127.0.0.1:60366_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:60366/collection1/ has no replicas
[junit4] 2> 2508888 INFO (coreZkRegister-8778-thread-1-processing-n:127.0.0.1:60366_ x:collection1 c:control_collection) [n:127.0.0.1:60366_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:60366/collection1/ shard1
[junit4] 2> 2508983 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2508985 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60356/solr ready
[junit4] 2> 2508985 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 2508985 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 2509047 INFO (coreZkRegister-8778-thread-1-processing-n:127.0.0.1:60366_ x:collection1 c:control_collection) [n:127.0.0.1:60366_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2509401 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-1-001/cores/collection1
[junit4] 2> 2509403 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-1-001
[junit4] 2> 2509405 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2509406 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@72d473ab{/,null,AVAILABLE}
[junit4] 2> 2509408 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7f27d31d{SSL,[ssl, http/1.1]}{127.0.0.1:60376}
[junit4] 2> 2509408 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.Server Started @2514956ms
[junit4] 2> 2509408 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=60376, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-1-001/cores}
[junit4] 2> 2509409 ERROR (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2509409 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2509409 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2509409 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2509409 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-09T17:27:39.362Z
[junit4] 2> 2509414 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2509414 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-1-001/solr.xml
[junit4] 2> 2509454 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60356/solr
[junit4] 2> 2509471 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60376_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2509481 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60376_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60376_
[junit4] 2> 2509485 INFO (zkCallback-2754-thread-3-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2509485 INFO (zkCallback-2758-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2509486 INFO (zkCallback-2764-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2509545 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60376_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-1-001/cores
[junit4] 2> 2509545 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60376_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2509551 INFO (OverseerStateUpdate-97254300212461573-127.0.0.1:60366_-n_0000000000) [n:127.0.0.1:60366_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2509676 INFO (zkCallback-2764-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2510566 WARN (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2510567 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2510580 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2510667 WARN (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2510669 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2510691 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2510691 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 2510691 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@24b1deb
[junit4] 2> 2510694 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=84.708984375, floorSegmentMB=0.6923828125, forceMergeDeletesPctAllowed=13.691854934993254, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2510708 WARN (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2510720 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2510721 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2510721 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2510721 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2510722 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=5.439453125, floorSegmentMB=0.2607421875, forceMergeDeletesPctAllowed=11.722461350720005, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 2510723 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@293addc2[collection1] main]
[junit4] 2> 2510725 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2510726 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2510726 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2510727 INFO (searcherExecutor-8797-thread-1-processing-n:127.0.0.1:60376_ x:collection1 c:collection1) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@293addc2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2510727 INFO (coreLoadExecutor-8796-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556068812120391680
[junit4] 2> 2510739 INFO (coreZkRegister-8791-thread-1-processing-n:127.0.0.1:60376_ x:collection1 c:collection1) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2510739 INFO (coreZkRegister-8791-thread-1-processing-n:127.0.0.1:60376_ x:collection1 c:collection1) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2510739 INFO (coreZkRegister-8791-thread-1-processing-n:127.0.0.1:60376_ x:collection1 c:collection1) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:60376/collection1/
[junit4] 2> 2510739 INFO (coreZkRegister-8791-thread-1-processing-n:127.0.0.1:60376_ x:collection1 c:collection1) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2510739 INFO (coreZkRegister-8791-thread-1-processing-n:127.0.0.1:60376_ x:collection1 c:collection1) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:60376/collection1/ has no replicas
[junit4] 2> 2510747 INFO (coreZkRegister-8791-thread-1-processing-n:127.0.0.1:60376_ x:collection1 c:collection1) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:60376/collection1/ shard1
[junit4] 2> 2510854 INFO (zkCallback-2764-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2510859 INFO (coreZkRegister-8791-thread-1-processing-n:127.0.0.1:60376_ x:collection1 c:collection1) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2510975 INFO (zkCallback-2764-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2511329 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-2-001/cores/collection1
[junit4] 2> 2511332 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-2-001
[junit4] 2> 2511335 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2511337 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4fb31aaa{/,null,AVAILABLE}
[junit4] 2> 2511341 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4fc04b73{SSL,[ssl, http/1.1]}{127.0.0.1:60385}
[junit4] 2> 2511341 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.Server Started @2516889ms
[junit4] 2> 2511341 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=60385, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-2-001/cores}
[junit4] 2> 2511341 ERROR (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2511342 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2511342 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2511342 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2511342 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-09T17:27:41.295Z
[junit4] 2> 2511348 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2511348 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-2-001/solr.xml
[junit4] 2> 2511374 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60356/solr
[junit4] 2> 2511393 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60385_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2511404 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60385_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60385_
[junit4] 2> 2511409 INFO (zkCallback-2754-thread-1-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2511409 INFO (zkCallback-2758-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2511409 INFO (zkCallback-2764-thread-1-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2511411 INFO (zkCallback-2770-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2511525 INFO (zkCallback-2764-thread-2-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2511532 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60385_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-2-001/cores
[junit4] 2> 2511532 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60385_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2511536 INFO (OverseerStateUpdate-97254300212461573-127.0.0.1:60366_-n_0000000000) [n:127.0.0.1:60366_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2511645 INFO (zkCallback-2764-thread-2-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2511645 INFO (zkCallback-2770-thread-2-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2512551 WARN (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2512552 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2512565 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2512680 WARN (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2512684 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2512707 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2512708 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 2512708 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@24b1deb
[junit4] 2> 2512714 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=84.708984375, floorSegmentMB=0.6923828125, forceMergeDeletesPctAllowed=13.691854934993254, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2512732 WARN (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2512786 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2512786 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2512787 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2512787 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2512788 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=5.439453125, floorSegmentMB=0.2607421875, forceMergeDeletesPctAllowed=11.722461350720005, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 2512799 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7ca772d5[collection1] main]
[junit4] 2> 2512801 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2512802 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2512802 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2512803 INFO (searcherExecutor-8808-thread-1-processing-n:127.0.0.1:60385_ x:collection1 c:collection1) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7ca772d5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2512804 INFO (coreLoadExecutor-8807-thread-1-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556068814298284032
[junit4] 2> 2512817 INFO (coreZkRegister-8802-thread-1-processing-n:127.0.0.1:60385_ x:collection1 c:collection1) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2512817 INFO (updateExecutor-2767-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2512818 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2512818 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2512818 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2512818 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2512819 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [https://127.0.0.1:60376/collection1/] and I am [https://127.0.0.1:60385/collection1/]
[junit4] 2> 2512822 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [https://127.0.0.1:60376]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:60385_&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2512849 INFO (qtp813063459-16744) [n:127.0.0.1:60376_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2512849 INFO (qtp813063459-16744) [n:127.0.0.1:60376_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 2512850 INFO (qtp813063459-16744) [n:127.0.0.1:60376_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:60385_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"https://127.0.0.1:60385","node_name":"127.0.0.1:60385_","state":"down"}
[junit4] 2> 2512929 INFO (zkCallback-2770-thread-2-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2512929 INFO (zkCallback-2764-thread-2-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2513585 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-3-001/cores/collection1
[junit4] 2> 2513587 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-3-001
[junit4] 2> 2513589 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2513591 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3dcf2380{/,null,AVAILABLE}
[junit4] 2> 2513592 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@222cab28{SSL,[ssl, http/1.1]}{127.0.0.1:60397}
[junit4] 2> 2513593 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.e.j.s.Server Started @2519141ms
[junit4] 2> 2513593 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=60397, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-3-001/cores}
[junit4] 2> 2513593 ERROR (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2513594 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2513594 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2513594 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2513594 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-09T17:27:43.547Z
[junit4] 2> 2513598 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2513598 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-3-001/solr.xml
[junit4] 2> 2513620 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60356/solr
[junit4] 2> 2513637 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60397_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2513645 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60397_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60397_
[junit4] 2> 2513649 INFO (zkCallback-2758-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2513649 INFO (zkCallback-2777-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2513649 INFO (zkCallback-2770-thread-2-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2513649 INFO (zkCallback-2754-thread-3-processing-n:127.0.0.1:60366_) [n:127.0.0.1:60366_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2513650 INFO (zkCallback-2764-thread-2-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2513706 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60397_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-3-001/cores
[junit4] 2> 2513706 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [n:127.0.0.1:60397_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2513710 INFO (OverseerStateUpdate-97254300212461573-127.0.0.1:60366_-n_0000000000) [n:127.0.0.1:60366_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2513816 INFO (zkCallback-2777-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2513816 INFO (zkCallback-2764-thread-2-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2513816 INFO (zkCallback-2770-thread-2-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2513851 INFO (qtp813063459-16744) [n:127.0.0.1:60376_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:60385_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"https://127.0.0.1:60385","node_name":"127.0.0.1:60385_","state":"recovering"}
[junit4] 2> 2513851 INFO (qtp813063459-16744) [n:127.0.0.1:60376_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2513851 INFO (qtp813063459-16744) [n:127.0.0.1:60376_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:60385_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1002
[junit4] 2> 2514729 WARN (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2514731 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2514743 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2514865 WARN (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2514869 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2514897 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2514897 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 2514897 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@24b1deb
[junit4] 2> 2514900 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=84.708984375, floorSegmentMB=0.6923828125, forceMergeDeletesPctAllowed=13.691854934993254, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2514909 WARN (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2514931 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2514931 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2514932 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2514933 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2514933 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=5.439453125, floorSegmentMB=0.2607421875, forceMergeDeletesPctAllowed=11.722461350720005, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 2514934 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@fd0c210[collection1] main]
[junit4] 2> 2514936 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2514937 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2514937 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2514938 INFO (searcherExecutor-8819-thread-1-processing-n:127.0.0.1:60397_ x:collection1 c:collection1) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@fd0c210[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2514938 INFO (coreLoadExecutor-8818-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556068816535945216
[junit4] 2> 2514946 INFO (coreZkRegister-8813-thread-1-processing-n:127.0.0.1:60397_ x:collection1 c:collection1) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2514946 INFO (updateExecutor-2774-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2514947 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2514948 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2514948 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2514948 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2514948 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [https://127.0.0.1:60376/collection1/] and I am [https://127.0.0.1:60397/collection1/]
[junit4] 2> 2514951 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [https://127.0.0.1:60376]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:60397_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2514958 INFO (qtp813063459-16742) [n:127.0.0.1:60376_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2514960 INFO (qtp813063459-16742) [n:127.0.0.1:60376_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 2514960 INFO (qtp813063459-16742) [n:127.0.0.1:60376_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:60397_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"https://127.0.0.1:60397","node_name":"127.0.0.1:60397_","state":"down"}
[junit4] 2> 2514977 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2514977 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30 for each attempt
[junit4] 2> 2514977 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 2515059 INFO (zkCallback-2770-thread-2-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2515059 INFO (zkCallback-2764-thread-2-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2515059 INFO (zkCallback-2777-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2515961 INFO (qtp813063459-16742) [n:127.0.0.1:60376_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:60397_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"https://127.0.0.1:60397","node_name":"127.0.0.1:60397_","state":"recovering"}
[junit4] 2> 2515961 INFO (qtp813063459-16742) [n:127.0.0.1:60376_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2515961 INFO (qtp813063459-16742) [n:127.0.0.1:60376_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:60397_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1002
[junit4] 2> 2520859 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [https://127.0.0.1:60376/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2520859 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=https://127.0.0.1:60385 START replicas=[https://127.0.0.1:60376/collection1/] nUpdates=1000
[junit4] 2> 2520864 INFO (qtp813063459-16745) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2520864 INFO (qtp813063459-16745) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2520865 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2520865 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2520865 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2520866 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2520866 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2520866 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2520866 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2520866 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2520866 INFO (recoveryExecutor-2768-thread-1-processing-n:127.0.0.1:60385_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2520871 INFO (zkCallback-2770-thread-2-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2520871 INFO (zkCallback-2764-thread-2-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2520871 INFO (zkCallback-2777-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2522963 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [https://127.0.0.1:60376/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2522963 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=https://127.0.0.1:60397 START replicas=[https://127.0.0.1:60376/collection1/] nUpdates=1000
[junit4] 2> 2522969 INFO (qtp813063459-16742) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2522970 INFO (qtp813063459-16742) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2522971 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2522971 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2522972 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2522972 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2522972 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2522972 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2522972 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2522972 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2522972 INFO (recoveryExecutor-2775-thread-1-processing-n:127.0.0.1:60397_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2522977 INFO (zkCallback-2777-thread-1-processing-n:127.0.0.1:60397_) [n:127.0.0.1:60397_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2522977 INFO (zkCallback-2764-thread-2-processing-n:127.0.0.1:60376_) [n:127.0.0.1:60376_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2522977 INFO (zkCallback-2770-thread-2-processing-n:127.0.0.1:60385_) [n:127.0.0.1:60385_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2523014 INFO (TEST-PeerSyncReplicationTest.test-seed#[8A0AC4DB45F62808]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 2523037 INFO (qtp486490392-16706) [n:127.0.0.1:60366_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2523037 INFO (qtp486490392-16706) [n:127.0.0.1:60366_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2523038 INFO (qtp486490392-16706) [n:127.0.0.1:60366_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2523038 INFO (qtp486490392-16706) [n:127.0.0.1:60366_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 1
[junit4] 2> 2523075 INFO (qtp813063459-16742) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2523075 INFO (qtp813063459-16742) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2523076 INFO (qtp813063459-16742) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2523081 INFO (qtp813063459-16742) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:60376/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 6
[junit4] 2> 2523085 INFO (qtp880549718-16773) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2523085 INFO (qtp880549718-16773) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2523085 INFO (qtp880549718-16773) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2523085 INFO (qtp880549718-16773) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:60376/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 2523086 INFO (qtp777167625-16811) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2523086 INFO (qtp777167625-16811) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2523087 INFO (qtp777167625-16811) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2523087 INFO (qtp777167625-16811) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:60376/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
[junit4] 2> 2523087 INFO (qtp813063459-16746) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={_stateVer_=collection1:10&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 45
[junit4] 2> 2523092 INFO (qtp813063459-16745) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2523098 INFO (qtp880549718-16776) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2523101 INFO (qtp777167625-16814) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2525110 INFO (qtp486490392-16707) [n:127.0.0.1:60366_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1556068827199963136)} 0 1
[junit4] 2> 2525116 INFO (qtp880549718-16769) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1556068827203108864&distrib.from=https://127.0.0.1:60376/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1556068827203108864)} 0 1
[junit4] 2> 2525116 INFO (qtp777167625-16807) [n:127.0.0.1:60397_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1556068827203108864&distrib.from=https://127.0.0.1:60376/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1556068827203108864)} 0 1
[junit4] 2> 2525116 INFO (qtp813063459-16747) [n:127.0.0.1:60376_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1556068827203108864)} 0 5
[junit4] 2> 2525121 INFO (qtp880549718-16771) [n:127.0.0.1:60385_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update pa
[...truncated too long message...]
nit4] 2> 2715825 INFO (Thread-3889) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60356 60356
[junit4] 2> 2718176 WARN (Thread-3889) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2> 2 /solr/overseer_elect/election/97254300212461578-127.0.0.1:60376_-n_0000000001
[junit4] 2> 2 /solr/collections/collection1/leader_elect/shard1/election/97254300212461578-core_node1-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 207 /solr/overseer/collection-queue-work
[junit4] 2> 41 /solr/overseer/queue
[junit4] 2> 13 /solr/overseer/queue-work
[junit4] 2> 6 /solr/collections
[junit4] 2> 5 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=PeerSyncReplicationTest -Dtests.method=test -Dtests.seed=8A0AC4DB45F62808 -Dtests.slow=true -Dtests.locale=ms-MY -Dtests.timezone=Pacific/Kiritimati -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] FAILURE 216s J1 | PeerSyncReplicationTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: timeout waiting to see all nodes active
[junit4] > at __randomizedtesting.SeedInfo.seed([8A0AC4DB45F62808:25EFB01EB0A45F0]:0)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:311)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:262)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:244)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:133)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2718260 INFO (SUITE-PeerSyncReplicationTest-seed#[8A0AC4DB45F62808]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_8A0AC4DB45F62808-001
[junit4] 2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=8, maxDocsPerChunk=990, blockSize=700), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=8, blockSize=700)), sim=RandomSimilarity(queryNorm=false): {}, locale=ms-MY, timezone=Pacific/Kiritimati
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=3,threads=1,free=204332488,total=518979584
[junit4] 2> NOTE: All tests run in this JVM: [LeaderElectionIntegrationTest, DistributedIntervalFacetingTest, SignatureUpdateProcessorFactoryTest, TestRandomFlRTGCloud, TestObjectReleaseTracker, DistributedFacetPivotWhiteBoxTest, HttpSolrCallGetCoreTest, ShowFileRequestHandlerTest, TestLeaderElectionZkExpiry, TestSolrCoreProperties, DefaultValueUpdateProcessorTest, TestFieldCacheSanityChecker, SpellingQueryConverterTest, InfoHandlerTest, TestManagedSynonymFilterFactory, TestBinaryResponseWriter, TestTrieFacet, TestFieldCacheSortRandom, SolrIndexMetricsTest, TestQueryUtils, TestNamedUpdateProcessors, CursorMarkTest, PreAnalyzedUpdateProcessorTest, TestJmxMonitoredMap, TestQueryTypes, DistributedFacetPivotSmallAdvancedTest, UpdateParamsTest, OverriddenZkACLAndCredentialsProvidersTest, TestGraphTermsQParserPlugin, SolrCoreMetricManagerTest, TestImpersonationWithHadoopAuth, TestSmileRequest, HdfsRestartWhileUpdatingTest, QueryParsingTest, StatsComponentTest, StatsReloadRaceTest, HdfsRecoverLeaseTest, TestDistribIDF, TestJoin, TestMiniSolrCloudClusterSSL, NotRequiredUniqueKeyTest, RequestLoggingTest, UtilsToolTest, BigEndianAscendingWordSerializerTest, TestSolrCoreParser, UUIDUpdateProcessorFallbackTest, TestFreeTextSuggestions, TestAddFieldRealTimeGet, BlockCacheTest, TestConfigSetProperties, TestSuggestSpellingConverter, TestWriterPerf, HdfsLockFactoryTest, DeleteNodeTest, TestUniqueKeyFieldResource, TestIndexSearcher, CollectionTooManyReplicasTest, PreAnalyzedFieldTest, DistributedFacetPivotLargeTest, TestSearchPerf, TestRecoveryHdfs, SpellPossibilityIteratorTest, BooleanFieldTest, TestZkChroot, SolrMetricReporterTest, TestEmbeddedSolrServerConstructors, TestLuceneMatchVersion, TestQueryWrapperFilter, DistributedExpandComponentTest, VersionInfoTest, ClassificationUpdateProcessorIntegrationTest, TestReversedWildcardFilterFactory, HdfsUnloadDistributedZkTest, DirectUpdateHandlerOptimizeTest, RulesTest, CloneFieldUpdateProcessorFactoryTest, BasicAuthIntegrationTest, TestCrossCoreJoin, TestPushWriter, MinimalSchemaTest, TestNoOpRegenerator, TestAuthenticationFramework, TestExportWriter, TestDynamicFieldResource, TestCorePropertiesReload, TestCloudPseudoReturnFields, TestDeleteCollectionOnDownNodes, TestCollapseQParserPlugin, CoreAdminRequestStatusTest, TestDocSet, TestComponentsName, EnumFieldTest, BufferStoreTest, FieldAnalysisRequestHandlerTest, RegexBoostProcessorTest, TestSolrCloudWithDelegationTokens, TestSolrQueryParserDefaultOperatorResource, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestInitQParser, ConvertedLegacyTest, OverseerModifyCollectionTest, TestIndexingPerformance, ExitableDirectoryReaderTest, TestOrdValues, TestRestoreCore, TestUseDocValuesAsStored, TestDefaultSearchFieldResource, HdfsCollectionsAPIDistributedZkTest, DeleteInactiveReplicaTest, UUIDFieldTest, TestSha256AuthenticationProvider, TestSearcherReuse, MultiThreadedOCPTest, BadIndexSchemaTest, TestBadConfig, TestSchemaManager, TestDistributedMissingSort, TestUtils, ReturnFieldsTest, SolrInfoMBeanTest, TestConfigSetsAPIZkFailure, TestExceedMaxTermLength, BasicFunctionalityTest, SpellCheckComponentTest, ResponseLogComponentTest, AnalyticsMergeStrategyTest, DistributedQueryComponentOptimizationTest, TestSolrIndexConfig, CollectionsAPIDistributedZkTest, TestStandardQParsers, FieldMutatingUpdateProcessorTest, TestComplexPhraseQParserPlugin, ResourceLoaderTest, TestSQLHandlerNonCloud, FullSolrCloudDistribCmdsTest, TestManagedSchemaAPI, RollingRestartTest, ExplicitHLLTest, ShufflingReplicaListTransformerTest, ScriptEngineTest, TestReplicationHandlerBackup, ConnectionReuseTest, TestSolrQueryParser, TestFieldCacheWithThreads, TestCollationFieldDocValues, TestReload, AtomicUpdatesTest, PKIAuthenticationIntegrationTest, TestNonDefinedSimilarityFactory, LeaderFailureAfterFreshStartTest, HLLSerializationTest, FileBasedSpellCheckerTest, CSVRequestHandlerTest, CleanupOldIndexTest, SliceStateTest, TestDistributedGrouping, ZkStateReaderTest, ConjunctionSolrSpellCheckerTest, SortByFunctionTest, TestSolrCloudSnapshots, ExternalFileFieldSortTest, PreAnalyzedFieldManagedSchemaCloudTest, LoggingHandlerTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, OverseerTest, BasicZkTest, RecoveryZkTest, TestReplicationHandler, ClusterStateUpdateTest, TestRandomFaceting, ZkSolrClientTest, ShardRoutingCustomTest, AutoCommitTest, TestFaceting, TestRecovery, TestHashPartitioner, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, TestRealTimeGet, TestStressReorder, DistributedTermsComponentTest, TestCoreContainer, SimpleFacetsTest, TestSolr4Spatial, SolrCmdDistributorTest, TestSort, TestFiltering, SoftAutoCommitTest, SimplePostToolTest, TestCoreDiscovery, CoreAdminHandlerTest, TestTrie, TestCSVLoader, WordBreakSolrSpellCheckerTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, TestRemoteStreaming, TestSolrDeletionPolicy1, TestSurroundQueryParser, LukeRequestHandlerTest, PrimitiveFieldTypeTest, TermsComponentTest, DocumentBuilderTest, PathHierarchyTokenizerFactoryTest, MoreLikeThisHandlerTest, SolrPluginUtilsTest, UpdateRequestProcessorFactoryTest, UniqFieldsUpdateProcessorFactoryTest, JsonLoaderTest, TestPartialUpdateDeduplication, TestPHPSerializedResponseWriter, TestLFUCache, TestSolrDeletionPolicy2, MultiTermTest, NumericFieldsTest, TestConfig, TestFuzzyAnalyzedSuggestions, DirectSolrConnectionTest, TestPhraseSuggestions, TestCodecSupport, TestXIncludeConfig, EchoParamsTest, TestSweetSpotSimilarityFactory, TestDFRSimilarityFactory, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestIBSimilarityFactory, TestFastWriter, URLClassifyProcessorTest, PluginInfoTest, TestFastLRUCache, DateMathParserTest, TestSystemIdResolver, DateFieldTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestLRUCache, CircularListTest, DistributedMLTComponentTest, TestRTGBase, SolrTestCaseJ4Test, TestHighlightDedupGrouping, TestSimpleTrackingShardHandler, TestJettySolrRunner, ActionThrottleTest, AliasIntegrationTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrBootstrapTest, CollectionStateFormat2Test, CollectionsAPISolrJTest, CreateCollectionCleanupTest, DeleteLastCustomShardedReplicaTest, DeleteShardTest, DistribCursorPagingTest, HttpPartitionTest, LeaderInitiatedRecoveryOnShardRestartTest, MigrateRouteKeyTest, PeerSyncReplicationTest]
[junit4] Completed [521/677 (1!)] on J1 in 215.96s, 1 test, 1 failure <<< FAILURES!
[...truncated 62909 lines...]
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3769
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3769/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC
1 tests failed.
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
timeout waiting to see all nodes active
Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
at __randomizedtesting.SeedInfo.seed([4A39EBB288C3A0B6:C26DD468263FCD4E]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 11365 lines...]
[junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/init-core-data-001
[junit4] 2> 1007049 INFO (SUITE-PeerSyncReplicationTest-seed#[4A39EBB288C3A0B6]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 1007049 INFO (SUITE-PeerSyncReplicationTest-seed#[4A39EBB288C3A0B6]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 1007051 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1007052 INFO (Thread-1634) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1007052 INFO (Thread-1634) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1007158 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.ZkTestServer start zk server on port:53202
[junit4] 2> 1007194 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1007198 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 1007201 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1007204 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1007207 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1007210 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1007214 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1007217 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1007220 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1007223 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1007226 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1007758 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/control-001/cores/collection1
[junit4] 2> 1007761 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1007763 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2802ca1a{/,null,AVAILABLE}
[junit4] 2> 1007765 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@73309173{HTTP/1.1,[http/1.1]}{127.0.0.1:53205}
[junit4] 2> 1007765 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.Server Started @1013347ms
[junit4] 2> 1007765 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/tempDir-001/control/data, hostContext=/, hostPort=53205, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/control-001/cores}
[junit4] 2> 1007766 ERROR (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1007766 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1007766 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1007766 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1007766 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-10T13:05:51.401Z
[junit4] 2> 1007771 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1007771 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/control-001/solr.xml
[junit4] 2> 1007800 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53202/solr
[junit4] 2> 1007852 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53205_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:53205_
[junit4] 2> 1007854 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53205_ ] o.a.s.c.Overseer Overseer (id=97258933497954308-127.0.0.1:53205_-n_0000000000) starting
[junit4] 2> 1007875 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53205_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53205_
[junit4] 2> 1007878 INFO (zkCallback-2021-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1008032 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53205_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/control-001/cores
[junit4] 2> 1008032 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53205_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1008047 INFO (OverseerStateUpdate-97258933497954308-127.0.0.1:53205_-n_0000000000) [n:127.0.0.1:53205_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1009065 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1009081 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1009301 WARN (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1009304 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1009326 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 1009327 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/control-001/cores/collection1/data/]
[junit4] 2> 1009327 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5e983ee5
[junit4] 2> 1009330 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=48.986328125, floorSegmentMB=1.4169921875, forceMergeDeletesPctAllowed=17.289735763156585, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6146725899110337
[junit4] 2> 1009337 WARN (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1009350 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1009350 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1009351 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1009351 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1009352 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=12.322265625, floorSegmentMB=2.0947265625, forceMergeDeletesPctAllowed=18.392260164660186, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6142252830410042
[junit4] 2> 1009353 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@33f03f73[collection1] main]
[junit4] 2> 1009355 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1009356 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1009356 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1009357 INFO (searcherExecutor-3568-thread-1-processing-n:127.0.0.1:53205_ x:collection1 c:control_collection) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@33f03f73[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1009357 INFO (coreLoadExecutor-3567-thread-1-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556142938382139392
[junit4] 2> 1009368 INFO (coreZkRegister-3560-thread-1-processing-n:127.0.0.1:53205_ x:collection1 c:control_collection) [n:127.0.0.1:53205_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1009368 INFO (coreZkRegister-3560-thread-1-processing-n:127.0.0.1:53205_ x:collection1 c:control_collection) [n:127.0.0.1:53205_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1009368 INFO (coreZkRegister-3560-thread-1-processing-n:127.0.0.1:53205_ x:collection1 c:control_collection) [n:127.0.0.1:53205_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53205/collection1/
[junit4] 2> 1009368 INFO (coreZkRegister-3560-thread-1-processing-n:127.0.0.1:53205_ x:collection1 c:control_collection) [n:127.0.0.1:53205_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1009368 INFO (coreZkRegister-3560-thread-1-processing-n:127.0.0.1:53205_ x:collection1 c:control_collection) [n:127.0.0.1:53205_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:53205/collection1/ has no replicas
[junit4] 2> 1009376 INFO (coreZkRegister-3560-thread-1-processing-n:127.0.0.1:53205_ x:collection1 c:control_collection) [n:127.0.0.1:53205_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53205/collection1/ shard1
[junit4] 2> 1009483 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1009487 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53202/solr ready
[junit4] 2> 1009487 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1009535 INFO (coreZkRegister-3560-thread-1-processing-n:127.0.0.1:53205_ x:collection1 c:control_collection) [n:127.0.0.1:53205_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1009810 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-1-001/cores/collection1
[junit4] 2> 1009812 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-1-001
[junit4] 2> 1009812 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1009814 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3b48078d{/,null,AVAILABLE}
[junit4] 2> 1009814 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6d8bda7f{HTTP/1.1,[http/1.1]}{127.0.0.1:53210}
[junit4] 2> 1009814 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.Server Started @1015396ms
[junit4] 2> 1009815 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=53210, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-1-001/cores}
[junit4] 2> 1009815 ERROR (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1009816 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1009816 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1009816 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1009816 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-10T13:05:53.451Z
[junit4] 2> 1009820 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1009820 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-1-001/solr.xml
[junit4] 2> 1009859 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53202/solr
[junit4] 2> 1009875 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53210_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1009885 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53210_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53210_
[junit4] 2> 1009888 INFO (zkCallback-2021-thread-2-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1009888 INFO (zkCallback-2025-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1009889 INFO (zkCallback-2030-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1010002 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53210_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-1-001/cores
[junit4] 2> 1010002 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53210_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1010006 INFO (OverseerStateUpdate-97258933497954308-127.0.0.1:53205_-n_0000000000) [n:127.0.0.1:53205_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1011022 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1011034 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1011126 WARN (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1011170 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1011201 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1011202 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 1011202 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5e983ee5
[junit4] 2> 1011205 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=48.986328125, floorSegmentMB=1.4169921875, forceMergeDeletesPctAllowed=17.289735763156585, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6146725899110337
[junit4] 2> 1011228 WARN (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1011247 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1011247 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1011248 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1011249 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1011250 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=12.322265625, floorSegmentMB=2.0947265625, forceMergeDeletesPctAllowed=18.392260164660186, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6142252830410042
[junit4] 2> 1011251 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@693f7f83[collection1] main]
[junit4] 2> 1011253 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1011254 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1011254 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1011255 INFO (searcherExecutor-3579-thread-1-processing-n:127.0.0.1:53210_ x:collection1 c:collection1) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@693f7f83[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1011256 INFO (coreLoadExecutor-3578-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556142940373385216
[junit4] 2> 1011267 INFO (coreZkRegister-3573-thread-1-processing-n:127.0.0.1:53210_ x:collection1 c:collection1) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1011267 INFO (coreZkRegister-3573-thread-1-processing-n:127.0.0.1:53210_ x:collection1 c:collection1) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1011267 INFO (coreZkRegister-3573-thread-1-processing-n:127.0.0.1:53210_ x:collection1 c:collection1) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53210/collection1/
[junit4] 2> 1011268 INFO (coreZkRegister-3573-thread-1-processing-n:127.0.0.1:53210_ x:collection1 c:collection1) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1011268 INFO (coreZkRegister-3573-thread-1-processing-n:127.0.0.1:53210_ x:collection1 c:collection1) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:53210/collection1/ has no replicas
[junit4] 2> 1011275 INFO (coreZkRegister-3573-thread-1-processing-n:127.0.0.1:53210_ x:collection1 c:collection1) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53210/collection1/ shard1
[junit4] 2> 1011435 INFO (coreZkRegister-3573-thread-1-processing-n:127.0.0.1:53210_ x:collection1 c:collection1) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1011607 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-2-001/cores/collection1
[junit4] 2> 1011608 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-2-001
[junit4] 2> 1011609 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1011610 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@143ebb29{/,null,AVAILABLE}
[junit4] 2> 1011611 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1f7ed30a{HTTP/1.1,[http/1.1]}{127.0.0.1:53214}
[junit4] 2> 1011611 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.Server Started @1017192ms
[junit4] 2> 1011611 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=53214, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-2-001/cores}
[junit4] 2> 1011612 ERROR (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1011612 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1011612 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1011612 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1011612 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-10T13:05:55.247Z
[junit4] 2> 1011616 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1011616 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-2-001/solr.xml
[junit4] 2> 1011672 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53202/solr
[junit4] 2> 1011690 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53214_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1011701 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53214_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53214_
[junit4] 2> 1011704 INFO (zkCallback-2025-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1011704 INFO (zkCallback-2030-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1011704 INFO (zkCallback-2021-thread-2-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1011706 INFO (zkCallback-2036-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1011789 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53214_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-2-001/cores
[junit4] 2> 1011789 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53214_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1011792 INFO (OverseerStateUpdate-97258933497954308-127.0.0.1:53205_-n_0000000000) [n:127.0.0.1:53205_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1012815 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1012830 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1012931 WARN (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1012933 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1012962 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1012962 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 1012962 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5e983ee5
[junit4] 2> 1012969 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=48.986328125, floorSegmentMB=1.4169921875, forceMergeDeletesPctAllowed=17.289735763156585, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6146725899110337
[junit4] 2> 1012992 WARN (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1013012 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1013012 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1013014 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1013014 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1013015 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=12.322265625, floorSegmentMB=2.0947265625, forceMergeDeletesPctAllowed=18.392260164660186, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6142252830410042
[junit4] 2> 1013016 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3341879a[collection1] main]
[junit4] 2> 1013019 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1013026 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1013026 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1013029 INFO (searcherExecutor-3590-thread-1-processing-n:127.0.0.1:53214_ x:collection1 c:collection1) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3341879a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1013033 INFO (coreLoadExecutor-3589-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556142942236704768
[junit4] 2> 1013040 INFO (coreZkRegister-3584-thread-1-processing-n:127.0.0.1:53214_ x:collection1 c:collection1) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 1013041 INFO (updateExecutor-2033-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 1013041 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 1013042 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 1013042 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 1013042 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 1013043 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:53210/collection1/] and I am [http://127.0.0.1:53214/collection1/]
[junit4] 2> 1013046 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:53210]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:53214_&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 1013050 INFO (qtp1595854443-9837) [n:127.0.0.1:53210_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 1013051 INFO (qtp1595854443-9837) [n:127.0.0.1:53210_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 1013051 INFO (qtp1595854443-9837) [n:127.0.0.1:53210_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:53214_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:53214","node_name":"127.0.0.1:53214_","state":"down"}
[junit4] 2> 1013633 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-3-001/cores/collection1
[junit4] 2> 1013634 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-3-001
[junit4] 2> 1013635 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1013637 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@25600c0d{/,null,AVAILABLE}
[junit4] 2> 1013638 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4111e0ca{HTTP/1.1,[http/1.1]}{127.0.0.1:53219}
[junit4] 2> 1013638 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.Server Started @1019219ms
[junit4] 2> 1013638 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=53219, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-3-001/cores}
[junit4] 2> 1013639 ERROR (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1013639 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1013639 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1013639 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1013639 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-10T13:05:57.274Z
[junit4] 2> 1013644 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1013644 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-3-001/solr.xml
[junit4] 2> 1013658 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53202/solr
[junit4] 2> 1013677 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53219_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1013687 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53219_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53219_
[junit4] 2> 1013691 INFO (zkCallback-2025-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1013691 INFO (zkCallback-2030-thread-1-processing-n:127.0.0.1:53210_) [n:127.0.0.1:53210_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1013691 INFO (zkCallback-2021-thread-2-processing-n:127.0.0.1:53205_) [n:127.0.0.1:53205_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1013692 INFO (zkCallback-2036-thread-1-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1013693 INFO (zkCallback-2043-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1013731 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53219_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-3-001/cores
[junit4] 2> 1013731 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [n:127.0.0.1:53219_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1013735 INFO (OverseerStateUpdate-97258933497954308-127.0.0.1:53205_-n_0000000000) [n:127.0.0.1:53205_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1014059 INFO (qtp1595854443-9837) [n:127.0.0.1:53210_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:53214_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:53214","node_name":"127.0.0.1:53214_","state":"recovering"}
[junit4] 2> 1014060 INFO (qtp1595854443-9837) [n:127.0.0.1:53210_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 1014060 INFO (qtp1595854443-9837) [n:127.0.0.1:53210_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:53214_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1009
[junit4] 2> 1014752 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1014764 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1014847 WARN (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1014849 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1014869 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1014870 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 1014870 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5e983ee5
[junit4] 2> 1014873 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=48.986328125, floorSegmentMB=1.4169921875, forceMergeDeletesPctAllowed=17.289735763156585, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6146725899110337
[junit4] 2> 1014881 WARN (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1014896 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1014897 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1014898 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1014898 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1014898 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=12.322265625, floorSegmentMB=2.0947265625, forceMergeDeletesPctAllowed=18.392260164660186, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6142252830410042
[junit4] 2> 1014900 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@39aea14f[collection1] main]
[junit4] 2> 1014902 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1014902 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1014902 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1014903 INFO (searcherExecutor-3601-thread-1-processing-n:127.0.0.1:53219_ x:collection1 c:collection1) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@39aea14f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1014904 INFO (coreLoadExecutor-3600-thread-1-processing-n:127.0.0.1:53219_) [n:127.0.0.1:53219_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556142944198590464
[junit4] 2> 1014909 INFO (coreZkRegister-3595-thread-1-processing-n:127.0.0.1:53219_ x:collection1 c:collection1) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 1014910 INFO (updateExecutor-2040-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 1014910 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 1014911 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 1014911 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 1014911 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 1014911 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:53210/collection1/] and I am [http://127.0.0.1:53219/collection1/]
[junit4] 2> 1014913 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:53210]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:53219_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 1014915 INFO (qtp1595854443-9841) [n:127.0.0.1:53210_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 1014915 INFO (qtp1595854443-9841) [n:127.0.0.1:53210_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 1014915 INFO (qtp1595854443-9841) [n:127.0.0.1:53210_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:53219_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:53219","node_name":"127.0.0.1:53219_","state":"down"}
[junit4] 2> 1015074 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 1015074 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30 for each attempt
[junit4] 2> 1015074 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 1015921 INFO (qtp1595854443-9841) [n:127.0.0.1:53210_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:53219_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:53219","node_name":"127.0.0.1:53219_","state":"recovering"}
[junit4] 2> 1015921 INFO (qtp1595854443-9841) [n:127.0.0.1:53210_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 1015921 INFO (qtp1595854443-9841) [n:127.0.0.1:53210_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:53219_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1006
[junit4] 2> 1021061 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:53210/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 1021061 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:53214 START replicas=[http://127.0.0.1:53210/collection1/] nUpdates=1000
[junit4] 2> 1021064 INFO (qtp1595854443-9838) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1021064 INFO (qtp1595854443-9838) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 1021065 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1021065 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 1021066 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1021066 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1021066 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1021066 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 1021066 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 1021066 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 1021066 INFO (recoveryExecutor-2034-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 1022927 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:53210/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 1022927 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:53219 START replicas=[http://127.0.0.1:53210/collection1/] nUpdates=1000
[junit4] 2> 1022929 INFO (qtp1595854443-9839) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1022929 INFO (qtp1595854443-9839) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1022930 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1022931 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 1022931 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1022931 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1022931 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1022931 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 1022931 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 1022931 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 1022931 INFO (recoveryExecutor-2041-thread-1-processing-n:127.0.0.1:53219_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 1023111 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 1023114 INFO (qtp975395389-9800) [n:127.0.0.1:53205_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1023114 INFO (qtp975395389-9800) [n:127.0.0.1:53205_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1023115 INFO (qtp975395389-9800) [n:127.0.0.1:53205_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1023115 INFO (qtp975395389-9800) [n:127.0.0.1:53205_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 1
[junit4] 2> 1023119 INFO (qtp1595854443-9842) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1023119 INFO (qtp1595854443-9842) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1023119 INFO (qtp1595854443-9842) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1023119 INFO (qtp1595854443-9842) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53210/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
[junit4] 2> 1023122 INFO (qtp1009627779-9900) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1023122 INFO (qtp1120343778-9865) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1023123 INFO (qtp1009627779-9900) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1023124 INFO (qtp1120343778-9865) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1023124 INFO (qtp1009627779-9900) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1023124 INFO (qtp1120343778-9865) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1023125 INFO (qtp1009627779-9900) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53210/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 1023125 INFO (qtp1120343778-9865) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53210/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 1023125 INFO (qtp1595854443-9841) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 8
[junit4] 2> 1023128 INFO (qtp1595854443-9835) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 1023132 INFO (qtp1120343778-9866) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 1023139 INFO (qtp1009627779-9901) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 1025145 INFO (qtp975395389-9801) [n:127.0.0.1:53205_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1556142954934960128)} 0 2
[junit4] 2> 1025150 INFO (qtp1120343778-9866) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1556142954938105856&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1556142954938105856)} 0 1
[junit4] 2> 1025150 INFO (qtp1009627779-9901) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1556142954938105856&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1556142954938105856)} 0 1
[junit4] 2> 1025150 INFO (qtp1595854443-9838) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1556142954938105856)} 0 4
[junit4] 2> 1025159 INFO (qtp1120343778-9868) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[0 (1556142954947543040)]} 0 1
[junit4] 2> 1025159 INFO (qtp1009627779-9903) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[0 (1556142954947543040)]} 0 1
[junit4] 2> 1025160 INFO (qtp1595854443-9839) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[0 (1556142954947543040)]} 0 5
[junit4] 2> 1025162 INFO (qtp1009627779-9904) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[1 (1556142954953834496)]} 0 0
[junit4] 2> 1025162 INFO (qtp1120343778-9869) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[1 (1556142954953834496)]} 0 0
[junit4] 2> 1025162 INFO (qtp1595854443-9840) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[1 (1556142954953834496)]} 0 1
[junit4] 2> 1025164 INFO (qtp1120343778-9869) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[2 (1556142954955931648)]} 0 0
[junit4] 2> 1025164 INFO (qtp1009627779-9904) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[2 (1556142954955931648)]} 0 0
[junit4] 2> 1025165 INFO (qtp1595854443-9837) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[2 (1556142954955931648)]} 0 1
[junit4] 2> 1025167 INFO (qtp1009627779-9898) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[3 (1556142954959077376)]} 0 0
[junit4] 2> 1025167 INFO (qtp1120343778-9863) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[3 (1556142954959077376)]} 0 0
[junit4] 2> 1025167 INFO (qtp1595854443-9842) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[3 (1556142954959077376)]} 0 1
[junit4] 2> 1025169 INFO (qtp1009627779-9900) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[4 (1556142954961174528)]} 0 0
[junit4] 2> 1025169 INFO (qtp1120343778-9865) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[4 (1556142954961174528)]} 0 0
[junit4] 2> 1025169 INFO (qtp1595854443-9841) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[4 (1556142954961174528)]} 0 1
[junit4] 2> 1025171 INFO (qtp1009627779-9902) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[5 (1556142954963271680)]} 0 0
[junit4] 2> 1025171 INFO (qtp1120343778-9867) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[5 (1556142954963271680)]} 0 0
[junit4] 2> 1025171 INFO (qtp1595854443-9835) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[5 (1556142954963271680)]} 0 1
[junit4] 2> 1025173 INFO (qtp1120343778-9866) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[6 (1556142954965368832)]} 0 0
[junit4] 2> 1025173 INFO (qtp1009627779-9902) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[6 (1556142954965368832)]} 0 0
[junit4] 2> 1025173 INFO (qtp1595854443-9838) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[6 (1556142954965368832)]} 0 1
[junit4] 2> 1025175 INFO (qtp1120343778-9866) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[7 (1556142954967465984)]} 0 0
[junit4] 2> 1025175 INFO (qtp1009627779-9902) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[7 (1556142954967465984)]} 0 0
[junit4] 2> 1025175 INFO (qtp1595854443-9839) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[7 (1556142954967465984)]} 0 1
[junit4] 2> 1025177 INFO (qtp1120343778-9866) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[8 (1556142954969563136)]} 0 0
[junit4] 2> 1025177 INFO (qtp1009627779-9902) [n:127.0.0.1:53219_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53210/collection1/&wt=javabin&version=2}{add=[8 (1556142954969563136)]} 0 0
[junit4] 2> 1025178 INFO (qtp1595854443-9840) [n:127.0.0.1:53210_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[8 (1556142954969563136)]} 0
[...truncated too long message...]
]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 53214
[junit4] 2> 1210592 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=565377
[junit4] 2> 1210592 WARN (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1] coreNodeName=[core_node2]
[junit4] 2> 1210596 WARN (updateExecutor-2047-thread-2-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Skipping recovery because Solr is shutdown
[junit4] 2> 1214367 INFO (recoveryExecutor-2048-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy RecoveryStrategy has been closed
[junit4] 2> 1214367 INFO (recoveryExecutor-2048-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[false]
[junit4] 2> 1214367 INFO (recoveryExecutor-2048-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@596060e9
[junit4] 2> 1214367 WARN (recoveryExecutor-2048-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1] coreNodeName=[core_node2]
[junit4] 2> 1214431 INFO (recoveryExecutor-2048-thread-1-processing-n:127.0.0.1:53214_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53214_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 1214431 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.Overseer Overseer (id=97258933497954321-127.0.0.1:53214_-n_0000000004) closing
[junit4] 2> 1214432 INFO (OverseerStateUpdate-97258933497954321-127.0.0.1:53214_-n_0000000004) [n:127.0.0.1:53214_ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:53214_
[junit4] 2> 1214435 WARN (zkCallback-2050-thread-4-processing-n:127.0.0.1:53214_) [n:127.0.0.1:53214_ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1214435 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 1214437 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@193108a{HTTP/1.1,[http/1.1]}{127.0.0.1:53214}
[junit4] 2> 1214438 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3f1784a9{/,null,UNAVAILABLE}
[junit4] 2> 1214441 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 53219
[junit4] 2> 1214445 INFO (TEST-PeerSyncReplicationTest.test-seed#[4A39EBB288C3A0B6]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53202 53202
[junit4] 2> 1214547 INFO (Thread-1634) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53202 53202
[junit4] 2> 1217240 WARN (Thread-1634) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 2 /solr/overseer_elect/election/97258933497954312-127.0.0.1:53210_-n_0000000001
[junit4] 2> 2 /solr/collections/collection1/leader_elect/shard1/election/97258933497954312-core_node1-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 207 /solr/overseer/collection-queue-work
[junit4] 2> 45 /solr/overseer/queue
[junit4] 2> 7 /solr/overseer/queue-work
[junit4] 2> 6 /solr/collections
[junit4] 2> 5 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=PeerSyncReplicationTest -Dtests.method=test -Dtests.seed=4A39EBB288C3A0B6 -Dtests.slow=true -Dtests.locale=es-PY -Dtests.timezone=Europe/Vilnius -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] FAILURE 210s J1 | PeerSyncReplicationTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: timeout waiting to see all nodes active
[junit4] > at __randomizedtesting.SeedInfo.seed([4A39EBB288C3A0B6:C26DD468263FCD4E]:0)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 1217245 INFO (SUITE-PeerSyncReplicationTest-seed#[4A39EBB288C3A0B6]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_4A39EBB288C3A0B6-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {other_tl1=Lucene50(blocksize=128), range_facet_l_dv=PostingsFormat(name=Asserting), rnd_s=Lucene50(blocksize=128), multiDefault=PostingsFormat(name=LuceneFixedGap), intDefault=Lucene50(blocksize=128), a_i1=PostingsFormat(name=LuceneFixedGap), range_facet_l=PostingsFormat(name=LuceneFixedGap), _version_=Lucene50(blocksize=128), a_t=PostingsFormat(name=LuceneFixedGap), id=PostingsFormat(name=Asserting), range_facet_i_dv=PostingsFormat(name=LuceneFixedGap), text=FSTOrd50, timestamp=PostingsFormat(name=LuceneFixedGap)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene70), range_facet_i_dv=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=264, maxMBSortInHeap=6.930022212166376, sim=RandomSimilarity(queryNorm=false): {}, locale=es-PY, timezone=Europe/Vilnius
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=3,threads=1,free=406568848,total=536870912
[junit4] 2> NOTE: All tests run in this JVM: [DefaultValueUpdateProcessorTest, TestSchemaVersionResource, TestSuggestSpellingConverter, TestRawTransformer, DataDrivenBlockJoinTest, TestSolrConfigHandlerConcurrent, DirectUpdateHandlerOptimizeTest, TestShardHandlerFactory, TestNRTOpen, CollectionTooManyReplicasTest, TestReversedWildcardFilterFactory, TestDynamicLoading, TestTrie, HdfsBasicDistributedZk2Test, TestCorePropertiesReload, DistributedFacetPivotLargeTest, BasicDistributedZk2Test, TestSmileRequest, TestRecoveryHdfs, TestBinaryResponseWriter, RequestHandlersTest, TestLeaderElectionZkExpiry, AnalyticsMergeStrategyTest, TestFieldResource, DistributedFacetPivotWhiteBoxTest, TestComplexPhraseQParserPlugin, TestFieldSortValues, ShowFileRequestHandlerTest, TestOrdValues, TestDistributedStatsComponentCardinality, TestRequestForwarding, TestWriterPerf, ExternalFileFieldSortTest, ClassificationUpdateProcessorIntegrationTest, TestRealTimeGet, CollectionsAPIDistributedZkTest, TestMiniSolrCloudCluster, TestAtomicUpdateErrorCases, AliasIntegrationTest, DistributedQueryComponentOptimizationTest, TestScoreJoinQPScore, TestUninvertingReader, HLLSerializationTest, HdfsRestartWhileUpdatingTest, HdfsLockFactoryTest, TestAuthenticationFramework, TestSSLRandomization, CoreAdminHandlerTest, StatsComponentTest, TestFieldCache, ScriptEngineTest, FullSolrCloudDistribCmdsTest, DeleteShardTest, UpdateRequestProcessorFactoryTest, LeaderElectionIntegrationTest, RAMDirectoryFactoryTest, TestCloudPseudoReturnFields, SuggestComponentContextFilterQueryTest, PKIAuthenticationIntegrationTest, TestSQLHandlerNonCloud, TestRandomFaceting, TestChildDocTransformer, TestRangeQuery, SolrJmxReporterTest, FacetPivotSmallTest, TestCollationFieldDocValues, TestPHPSerializedResponseWriter, BadIndexSchemaTest, HttpSolrCallGetCoreTest, TestJsonFacets, TestQueryWrapperFilter, TestRebalanceLeaders, HighlighterConfigTest, TestStressLiveNodes, TestTolerantUpdateProcessorRandomCloud, ReturnFieldsTest, DeleteNodeTest, MinimalSchemaTest, StatsReloadRaceTest, TestFieldCacheSanityChecker, TestRestoreCore, TestHdfsCloudBackupRestore, TestConfigSetsAPIZkFailure, TestSolrIndexConfig, HdfsUnloadDistributedZkTest, ConfigSetsAPITest, DistributedTermsComponentTest, TestBadConfig, TestConfigReload, SpellCheckComponentTest, SecurityConfHandlerTest, PeerSyncReplicationTest]
[junit4] Completed [221/677 (1!)] on J1 in 210.22s, 1 test, 1 failure <<< FAILURES!
[...truncated 63905 lines...]
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3768
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3768/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC
1 tests failed.
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
timeout waiting to see all nodes active
Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
at __randomizedtesting.SeedInfo.seed([66CC40C8C86C653B:EE987F12669008C3]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12288 lines...]
[junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/init-core-data-001
[junit4] 2> 2434082 INFO (SUITE-PeerSyncReplicationTest-seed#[66CC40C8C86C653B]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 2434082 INFO (SUITE-PeerSyncReplicationTest-seed#[66CC40C8C86C653B]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 2434084 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2434084 INFO (Thread-3750) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2434084 INFO (Thread-3750) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2434194 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.ZkTestServer start zk server on port:50302
[junit4] 2> 2434226 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2434234 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 2434239 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2434244 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2434248 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2434251 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2434257 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2434260 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2434266 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2434269 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2434272 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2437718 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/control-001/cores/collection1
[junit4] 2> 2437720 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2437722 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@dbbdd49{/,null,AVAILABLE}
[junit4] 2> 2437723 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2ed9cd73{HTTP/1.1,[http/1.1]}{127.0.0.1:50305}
[junit4] 2> 2437723 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.Server Started @2444051ms
[junit4] 2> 2437723 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/tempDir-001/control/data, hostContext=/, hostPort=50305, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/control-001/cores}
[junit4] 2> 2437724 ERROR (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2437724 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2437724 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2437724 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2437724 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-10T03:29:38.767Z
[junit4] 2> 2437729 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2437729 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/control-001/solr.xml
[junit4] 2> 2437739 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50302/solr
[junit4] 2> 2437786 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50305_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:50305_
[junit4] 2> 2437789 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50305_ ] o.a.s.c.Overseer Overseer (id=97256667554119684-127.0.0.1:50305_-n_0000000000) starting
[junit4] 2> 2437803 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50305_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50305_
[junit4] 2> 2437808 INFO (OverseerStateUpdate-97256667554119684-127.0.0.1:50305_-n_0000000000) [n:127.0.0.1:50305_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2437836 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50305_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/control-001/cores
[junit4] 2> 2437836 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50305_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2437842 INFO (OverseerStateUpdate-97256667554119684-127.0.0.1:50305_-n_0000000000) [n:127.0.0.1:50305_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2438863 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2438880 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2438977 WARN (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2438979 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2439004 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 2439006 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/control-001/cores/collection1/data/]
[junit4] 2> 2439006 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5df7ccfd
[junit4] 2> 2439009 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=620621274, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2439030 WARN (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2439045 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2439045 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2439046 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2439046 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2439046 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=44.421875, floorSegmentMB=1.451171875, forceMergeDeletesPctAllowed=19.653727303322867, segmentsPerTier=34.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2439047 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4cd3c4b7[collection1] main]
[junit4] 2> 2439049 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2439050 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2439050 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2439051 INFO (searcherExecutor-8096-thread-1-processing-n:127.0.0.1:50305_ x:collection1 c:control_collection) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4cd3c4b7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2439051 INFO (coreLoadExecutor-8095-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556106686071046144
[junit4] 2> 2439064 INFO (coreZkRegister-8088-thread-1-processing-n:127.0.0.1:50305_ x:collection1 c:control_collection) [n:127.0.0.1:50305_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2439064 INFO (coreZkRegister-8088-thread-1-processing-n:127.0.0.1:50305_ x:collection1 c:control_collection) [n:127.0.0.1:50305_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2439064 INFO (coreZkRegister-8088-thread-1-processing-n:127.0.0.1:50305_ x:collection1 c:control_collection) [n:127.0.0.1:50305_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:50305/collection1/
[junit4] 2> 2439064 INFO (coreZkRegister-8088-thread-1-processing-n:127.0.0.1:50305_ x:collection1 c:control_collection) [n:127.0.0.1:50305_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2439065 INFO (coreZkRegister-8088-thread-1-processing-n:127.0.0.1:50305_ x:collection1 c:control_collection) [n:127.0.0.1:50305_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:50305/collection1/ has no replicas
[junit4] 2> 2439074 INFO (coreZkRegister-8088-thread-1-processing-n:127.0.0.1:50305_ x:collection1 c:control_collection) [n:127.0.0.1:50305_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:50305/collection1/ shard1
[junit4] 2> 2439143 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2439145 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50302/solr ready
[junit4] 2> 2439145 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 2439145 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 2439186 INFO (coreZkRegister-8088-thread-1-processing-n:127.0.0.1:50305_ x:collection1 c:control_collection) [n:127.0.0.1:50305_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2439603 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-1-001/cores/collection1
[junit4] 2> 2439606 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-1-001
[junit4] 2> 2439607 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2439609 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@67ec134c{/,null,AVAILABLE}
[junit4] 2> 2439609 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@776d0000{HTTP/1.1,[http/1.1]}{127.0.0.1:50311}
[junit4] 2> 2439609 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.Server Started @2445937ms
[junit4] 2> 2439610 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=50311, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-1-001/cores}
[junit4] 2> 2439610 ERROR (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2439610 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2439610 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2439611 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2439611 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-10T03:29:40.654Z
[junit4] 2> 2439615 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2439616 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-1-001/solr.xml
[junit4] 2> 2439696 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50302/solr
[junit4] 2> 2439717 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50311_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2439727 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50311_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50311_
[junit4] 2> 2439730 INFO (zkCallback-2869-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2439731 INFO (zkCallback-2875-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2439732 INFO (zkCallback-2865-thread-3-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2439841 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50311_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-1-001/cores
[junit4] 2> 2439841 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50311_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2439848 INFO (OverseerStateUpdate-97256667554119684-127.0.0.1:50305_-n_0000000000) [n:127.0.0.1:50305_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2439955 INFO (zkCallback-2875-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2440870 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2440882 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2440950 WARN (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2440952 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2440977 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2440977 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 2440978 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5df7ccfd
[junit4] 2> 2440981 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=620621274, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2441013 WARN (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2441031 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2441031 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2441034 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2441034 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2441035 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=44.421875, floorSegmentMB=1.451171875, forceMergeDeletesPctAllowed=19.653727303322867, segmentsPerTier=34.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2441036 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@f28f594[collection1] main]
[junit4] 2> 2441038 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2441039 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2441039 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2441039 INFO (searcherExecutor-8107-thread-1-processing-n:127.0.0.1:50311_ x:collection1 c:collection1) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@f28f594[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2441040 INFO (coreLoadExecutor-8106-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556106688156663808
[junit4] 2> 2441053 INFO (coreZkRegister-8101-thread-1-processing-n:127.0.0.1:50311_ x:collection1 c:collection1) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2441053 INFO (coreZkRegister-8101-thread-1-processing-n:127.0.0.1:50311_ x:collection1 c:collection1) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2441053 INFO (coreZkRegister-8101-thread-1-processing-n:127.0.0.1:50311_ x:collection1 c:collection1) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:50311/collection1/
[junit4] 2> 2441053 INFO (coreZkRegister-8101-thread-1-processing-n:127.0.0.1:50311_ x:collection1 c:collection1) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2441053 INFO (coreZkRegister-8101-thread-1-processing-n:127.0.0.1:50311_ x:collection1 c:collection1) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:50311/collection1/ has no replicas
[junit4] 2> 2441062 INFO (coreZkRegister-8101-thread-1-processing-n:127.0.0.1:50311_ x:collection1 c:collection1) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:50311/collection1/ shard1
[junit4] 2> 2441173 INFO (zkCallback-2875-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2441223 INFO (coreZkRegister-8101-thread-1-processing-n:127.0.0.1:50311_ x:collection1 c:collection1) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2441337 INFO (zkCallback-2875-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2441586 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-2-001/cores/collection1
[junit4] 2> 2441587 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-2-001
[junit4] 2> 2441588 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2441591 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5f84848d{/,null,AVAILABLE}
[junit4] 2> 2441591 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@54c1c84c{HTTP/1.1,[http/1.1]}{127.0.0.1:50315}
[junit4] 2> 2441592 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.Server Started @2447919ms
[junit4] 2> 2441592 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=50315, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-2-001/cores}
[junit4] 2> 2441592 ERROR (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2441593 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2441593 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2441593 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2441593 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-10T03:29:42.636Z
[junit4] 2> 2441600 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2441600 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-2-001/solr.xml
[junit4] 2> 2441659 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50302/solr
[junit4] 2> 2441682 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50315_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2441694 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50315_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50315_
[junit4] 2> 2441698 INFO (zkCallback-2865-thread-1-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2441698 INFO (zkCallback-2869-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2441699 INFO (zkCallback-2875-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2441701 INFO (zkCallback-2881-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2441809 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50315_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-2-001/cores
[junit4] 2> 2441809 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50315_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2441814 INFO (zkCallback-2875-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2441814 INFO (zkCallback-2881-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2441819 INFO (OverseerStateUpdate-97256667554119684-127.0.0.1:50305_-n_0000000000) [n:127.0.0.1:50305_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2441924 INFO (zkCallback-2875-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2441924 INFO (zkCallback-2881-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2442839 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2442854 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2443015 WARN (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2443017 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2443044 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2443044 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 2443045 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5df7ccfd
[junit4] 2> 2443183 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=620621274, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2443208 WARN (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2443238 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2443238 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2443239 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2443239 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2443242 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=44.421875, floorSegmentMB=1.451171875, forceMergeDeletesPctAllowed=19.653727303322867, segmentsPerTier=34.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2443244 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@27867b58[collection1] main]
[junit4] 2> 2443247 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2443248 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2443248 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2443250 INFO (searcherExecutor-8118-thread-1-processing-n:127.0.0.1:50315_ x:collection1 c:collection1) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@27867b58[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2443250 INFO (coreLoadExecutor-8117-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556106690474016768
[junit4] 2> 2443261 INFO (coreZkRegister-8112-thread-1-processing-n:127.0.0.1:50315_ x:collection1 c:collection1) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2443263 INFO (updateExecutor-2878-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2443271 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2443272 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2443272 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2443272 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2443272 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:50311/collection1/] and I am [http://127.0.0.1:50315/collection1/]
[junit4] 2> 2443277 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:50311]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:50315_&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2443287 INFO (qtp610235088-16719) [n:127.0.0.1:50311_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2443290 INFO (qtp610235088-16719) [n:127.0.0.1:50311_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 2443290 INFO (qtp610235088-16719) [n:127.0.0.1:50311_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:50315_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:50315","node_name":"127.0.0.1:50315_","state":"down"}
[junit4] 2> 2443402 INFO (zkCallback-2875-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2443402 INFO (zkCallback-2881-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2443765 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-3-001/cores/collection1
[junit4] 2> 2443767 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-3-001
[junit4] 2> 2443768 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2443773 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3923c392{/,null,AVAILABLE}
[junit4] 2> 2443774 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5b007fe1{HTTP/1.1,[http/1.1]}{127.0.0.1:50320}
[junit4] 2> 2443774 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.Server Started @2450101ms
[junit4] 2> 2443774 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=50320, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-3-001/cores}
[junit4] 2> 2443775 ERROR (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2443775 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2443776 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2443776 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2443776 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-10T03:29:44.819Z
[junit4] 2> 2443782 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2443782 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-3-001/solr.xml
[junit4] 2> 2443840 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50302/solr
[junit4] 2> 2443863 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50320_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2443876 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50320_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50320_
[junit4] 2> 2443885 INFO (zkCallback-2869-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2443885 INFO (zkCallback-2865-thread-2-processing-n:127.0.0.1:50305_) [n:127.0.0.1:50305_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2443885 INFO (zkCallback-2875-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2443886 INFO (zkCallback-2888-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2443888 INFO (zkCallback-2881-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2444005 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50320_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-3-001/cores
[junit4] 2> 2444005 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [n:127.0.0.1:50320_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2444010 INFO (zkCallback-2875-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2444010 INFO (zkCallback-2881-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2444024 INFO (OverseerStateUpdate-97256667554119684-127.0.0.1:50305_-n_0000000000) [n:127.0.0.1:50305_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2444133 INFO (zkCallback-2875-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2444133 INFO (zkCallback-2881-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2444133 INFO (zkCallback-2888-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2444293 INFO (qtp610235088-16719) [n:127.0.0.1:50311_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:50315_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:50315","node_name":"127.0.0.1:50315_","state":"recovering"}
[junit4] 2> 2444293 INFO (qtp610235088-16719) [n:127.0.0.1:50311_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2444293 INFO (qtp610235088-16719) [n:127.0.0.1:50311_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:50315_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1006
[junit4] 2> 2445044 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2445060 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2445157 WARN (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2445159 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2445192 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2445192 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 2445193 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5df7ccfd
[junit4] 2> 2445199 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=620621274, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2445213 WARN (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2445238 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2445238 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2445239 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2445240 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2445240 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=44.421875, floorSegmentMB=1.451171875, forceMergeDeletesPctAllowed=19.653727303322867, segmentsPerTier=34.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2445242 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@df51beb[collection1] main]
[junit4] 2> 2445244 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2445246 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2445246 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2445248 INFO (searcherExecutor-8129-thread-1-processing-n:127.0.0.1:50320_ x:collection1 c:collection1) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@df51beb[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2445249 INFO (coreLoadExecutor-8128-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1556106692570120192
[junit4] 2> 2445255 INFO (coreZkRegister-8123-thread-1-processing-n:127.0.0.1:50320_ x:collection1 c:collection1) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2445256 INFO (updateExecutor-2885-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2445258 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2445258 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2445258 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2445258 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2445258 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:50311/collection1/] and I am [http://127.0.0.1:50320/collection1/]
[junit4] 2> 2445262 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:50311]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:50320_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2445264 INFO (qtp610235088-16722) [n:127.0.0.1:50311_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2445264 INFO (qtp610235088-16722) [n:127.0.0.1:50311_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 2445264 INFO (qtp610235088-16722) [n:127.0.0.1:50311_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:50320_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:50320","node_name":"127.0.0.1:50320_","state":"down"}
[junit4] 2> 2445371 INFO (zkCallback-2875-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2445371 INFO (zkCallback-2888-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2445371 INFO (zkCallback-2881-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2445632 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2445632 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30 for each attempt
[junit4] 2> 2445632 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 2446271 INFO (qtp610235088-16722) [n:127.0.0.1:50311_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:50320_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:50320","node_name":"127.0.0.1:50320_","state":"recovering"}
[junit4] 2> 2446271 INFO (qtp610235088-16722) [n:127.0.0.1:50311_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2446271 INFO (qtp610235088-16722) [n:127.0.0.1:50311_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:50320_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1007
[junit4] 2> 2451295 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:50311/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2451295 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:50315 START replicas=[http://127.0.0.1:50311/collection1/] nUpdates=1000
[junit4] 2> 2451298 INFO (qtp610235088-16717) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2451298 INFO (qtp610235088-16717) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 2451300 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2451300 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2451300 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2451300 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2451301 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2451301 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2451301 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2451301 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2451301 INFO (recoveryExecutor-2879-thread-1-processing-n:127.0.0.1:50315_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2451308 INFO (zkCallback-2881-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2451308 INFO (zkCallback-2875-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2451308 INFO (zkCallback-2888-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2453289 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:50311/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2453289 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:50320 START replicas=[http://127.0.0.1:50311/collection1/] nUpdates=1000
[junit4] 2> 2453291 INFO (qtp610235088-16719) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2453291 INFO (qtp610235088-16719) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2453292 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2453292 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2453292 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2453292 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2453293 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2453293 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2453293 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2453293 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2453293 INFO (recoveryExecutor-2886-thread-1-processing-n:127.0.0.1:50320_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2453300 INFO (zkCallback-2875-thread-1-processing-n:127.0.0.1:50311_) [n:127.0.0.1:50311_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2453301 INFO (zkCallback-2881-thread-1-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2453301 INFO (zkCallback-2888-thread-1-processing-n:127.0.0.1:50320_) [n:127.0.0.1:50320_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2453682 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 2453686 INFO (qtp1921313443-16680) [n:127.0.0.1:50305_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2453686 INFO (qtp1921313443-16680) [n:127.0.0.1:50305_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2453688 INFO (qtp1921313443-16680) [n:127.0.0.1:50305_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2453688 INFO (qtp1921313443-16680) [n:127.0.0.1:50305_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 1
[junit4] 2> 2453694 INFO (qtp610235088-16722) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2453694 INFO (qtp610235088-16722) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2453696 INFO (qtp610235088-16722) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2453697 INFO (qtp610235088-16722) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:50311/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 3
[junit4] 2> 2453697 INFO (qtp1805051827-16748) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2453697 INFO (qtp1805051827-16748) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2453697 INFO (qtp463790373-16783) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2453698 INFO (qtp463790373-16783) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2453698 INFO (qtp1805051827-16748) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2453698 INFO (qtp1805051827-16748) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:50311/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
[junit4] 2> 2453698 INFO (qtp463790373-16783) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2453698 INFO (qtp463790373-16783) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:50311/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
[junit4] 2> 2453699 INFO (qtp610235088-16720) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={_stateVer_=collection1:12&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 8
[junit4] 2> 2453702 INFO (qtp610235088-16723) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2453704 INFO (qtp1805051827-16750) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2453707 INFO (qtp463790373-16785) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2455718 INFO (qtp1921313443-16681) [n:127.0.0.1:50305_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1556106703545565184)} 0 2
[junit4] 2> 2455725 INFO (qtp463790373-16786) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1556106703548710912&distrib.from=http://127.0.0.1:50311/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1556106703548710912)} 0 1
[junit4] 2> 2455726 INFO (qtp1805051827-16751) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1556106703548710912&distrib.from=http://127.0.0.1:50311/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1556106703548710912)} 0 1
[junit4] 2> 2455726 INFO (qtp610235088-16717) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1556106703548710912)} 0 6
[junit4] 2> 2455731 INFO (qtp463790373-16787) [n:127.0.0.1:50320_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:50311/collection1/&wt=javabin&version=2}{add=[0 (1556106703557099520)]} 0 1
[junit4] 2> 2455731 INFO (qtp1805051827-16752) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:50311/collection1/&wt=javabin&version=2}{add=[0 (1556106703557099520)]} 0 1
[junit4] 2> 2455732 INFO (qtp610235088-16721) [n:127.0.0.1:50311_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={_stateVer_=collection1:
[...truncated too long message...]
veryStrategy Stopping recovery for core=[collection1] coreNodeName=[core_node2]
[junit4] 2> 2646773 INFO (zkCallback-2895-thread-3-processing-n:127.0.0.1:50315_) [n:127.0.0.1:50315_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 2646774 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 2646776 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@1ac3f659{HTTP/1.1,[http/1.1]}{127.0.0.1:50315}
[junit4] 2> 2646776 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@126ce9ae{/,null,UNAVAILABLE}
[junit4] 2> 2646777 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 50320
[junit4] 2> 2646779 INFO (TEST-PeerSyncReplicationTest.test-seed#[66CC40C8C86C653B]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50302 50302
[junit4] 2> 2646805 INFO (Thread-3750) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50302 50302
[junit4] 2> 2649126 WARN (Thread-3750) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2> 2 /solr/collections/collection1/leader_elect/shard1/election/97256667554119689-core_node1-n_0000000000
[junit4] 2> 2 /solr/overseer_elect/election/97256667554119689-127.0.0.1:50311_-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 207 /solr/overseer/collection-queue-work
[junit4] 2> 43 /solr/overseer/queue
[junit4] 2> 13 /solr/overseer/queue-work
[junit4] 2> 6 /solr/collections
[junit4] 2> 5 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=PeerSyncReplicationTest -Dtests.method=test -Dtests.seed=66CC40C8C86C653B -Dtests.slow=true -Dtests.locale=el -Dtests.timezone=Pacific/Norfolk -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 215s J1 | PeerSyncReplicationTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: timeout waiting to see all nodes active
[junit4] > at __randomizedtesting.SeedInfo.seed([66CC40C8C86C653B:EE987F12669008C3]:0)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2649171 INFO (SUITE-PeerSyncReplicationTest-seed#[66CC40C8C86C653B]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_66CC40C8C86C653B-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {}, locale=el, timezone=Pacific/Norfolk
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=3,threads=1,free=150577016,total=536870912
[junit4] 2> NOTE: All tests run in this JVM: [SolrMetricReporterTest, TestDynamicLoading, TestJoin, TestChildDocTransformer, TestDownShardTolerantSearch, TestFieldCacheSanityChecker, TestSearcherReuse, TestCrossCoreJoin, RankQueryTest, ResponseLogComponentTest, HdfsRecoverLeaseTest, FieldAnalysisRequestHandlerTest, TestDistributedStatsComponentCardinality, StatsComponentTest, CollectionsAPIDistributedZkTest, BasicFunctionalityTest, TestDistributedGrouping, BasicAuthIntegrationTest, TestDocSet, HdfsCollectionsAPIDistributedZkTest, FileBasedSpellCheckerTest, OpenCloseCoreStressTest, NoCacheHeaderTest, TestTrie, TestRequestForwarding, TestBadConfig, TestSolrCoreParser, TestPHPSerializedResponseWriter, OverseerModifyCollectionTest, TestBinaryResponseWriter, LoggingHandlerTest, TestQueryUtils, AtomicUpdatesTest, TestDistributedMissingSort, CloudMLTQParserTest, OverseerStatusTest, TestTrieFacet, DefaultValueUpdateProcessorTest, TestSolrCoreProperties, TestWriterPerf, TestSchemaResource, TestNRTOpen, TestSimpleTrackingShardHandler, BJQParserTest, TestFuzzyAnalyzedSuggestions, TestSolrIndexConfig, SolrJmxReporterTest, TestDynamicFieldResource, CoreAdminHandlerTest, AnalysisErrorHandlingTest, CoreSorterTest, UpdateRequestProcessorFactoryTest, ZkStateReaderTest, TestRandomFlRTGCloud, ScriptEngineTest, SolrGangliaReporterTest, SolrIndexMetricsTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, VersionInfoTest, DistanceFunctionTest, BadIndexSchemaTest, DistributedExpandComponentTest, RAMDirectoryFactoryTest, UUIDFieldTest, TestFieldSortValues, DistributedVersionInfoTest, TestSQLHandlerNonCloud, HLLSerializationTest, DistributedFacetPivotSmallAdvancedTest, RequestLoggingTest, RulesTest, LeaderFailureAfterFreshStartTest, TestEmbeddedSolrServerConstructors, CurrencyFieldOpenExchangeTest, TestRebalanceLeaders, OverriddenZkACLAndCredentialsProvidersTest, ConjunctionSolrSpellCheckerTest, SpellPossibilityIteratorTest, TestDFRSimilarityFactory, RegexBoostProcessorTest, RemoteQueryErrorTest, TestSSLRandomization, BlockJoinFacetRandomTest, CollectionReloadTest, TestCodecSupport, QueryParsingTest, TermVectorComponentTest, TestRawTransformer, QueryResultKeyTest, TestSchemaVersionResource, NotRequiredUniqueKeyTest, HdfsThreadLeakTest, EchoParamsTest, CreateCollectionCleanupTest, SpellCheckCollatorTest, HdfsLockFactoryTest, TestMiniSolrCloudClusterSSL, TestObjectReleaseTracker, DateRangeFieldTest, TestAuthenticationFramework, SortByFunctionTest, TestCorePropertiesReload, TestDistribIDF, TestFieldCache, JsonLoaderTest, TestNonDefinedSimilarityFactory, SolrCmdDistributorTest, FacetPivotSmallTest, BasicZkTest, PreAnalyzedUpdateProcessorTest, TestManagedSchemaAPI, BadComponentTest, TestTolerantUpdateProcessorRandomCloud, DistributedFacetPivotWhiteBoxTest, GraphQueryTest, ExplicitHLLTest, TestZkChroot, TestUseDocValuesAsStored, SolrCloudExampleTest, TestSolrCloudWithDelegationTokens, BasicDistributedZk2Test, TestSearchPerf, DeleteShardTest, TestPivotHelperCode, TestExactStatsCache, PKIAuthenticationIntegrationTest, TestConfig, TestBlendedInfixSuggestions, ClusterStateTest, TestFastWriter, TestConfigSetProperties, TestDefaultSearchFieldResource, DistributedIntervalFacetingTest, TestSolr4Spatial, TestPKIAuthenticationPlugin, TestReplicationHandlerBackup, TestFieldCacheSortRandom, TestInitQParser, IndexSchemaTest, TestAddFieldRealTimeGet, TestLuceneMatchVersion, TestSolrConfigHandlerConcurrent, DistribJoinFromCollectionTest, ReplicationFactorTest, TestLocalFSCloudBackupRestore, TestLMDirichletSimilarityFactory, TestMiniSolrCloudCluster, RollingRestartTest, TestDeleteCollectionOnDownNodes, MultiThreadedOCPTest, CdcrVersionReplicationTest, PreAnalyzedFieldManagedSchemaCloudTest, TestNoOpRegenerator, TestSolrCloudWithSecureImpersonation, ShufflingReplicaListTransformerTest, HdfsUnloadDistributedZkTest, TestPushWriter, CleanupOldIndexTest, TestJsonFacets, AliasIntegrationTest, TestIndexSearcher, TestExceedMaxTermLength, CursorMarkTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, LeaderElectionIntegrationTest, FullSolrCloudDistribCmdsTest, TestRandomFaceting, ZkSolrClientTest, TestRandomDVFaceting, ShardRoutingCustomTest, AutoCommitTest, TestFaceting, TestRecovery, DistributedSpellCheckComponentTest, TestRealTimeGet, TestReload, DistributedTermsComponentTest, SpellCheckComponentTest, PeerSyncTest, ConvertedLegacyTest, TestSort, TestFunctionQuery, HighlighterTest, SoftAutoCommitTest, SignatureUpdateProcessorFactoryTest, TestFoldingMultitermQuery, DocValuesTest, SolrInfoMBeanTest, DisMaxRequestHandlerTest, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, PrimitiveFieldTypeTest, RequestHandlersTest, TestValueSourceCache, MoreLikeThisHandlerTest, IndexSchemaRuntimeFieldTest, SolrPluginUtilsTest, ReturnFieldsTest, TestAnalyzedSuggestions, PingRequestHandlerTest, UpdateParamsTest, TestStressRecovery, PreAnalyzedFieldTest, TestSystemIdResolver, PrimUtilsTest, TestSuggestSpellingConverter, SpellingQueryConverterTest, DOMUtilTest, TestLRUCache, TestUtils, SliceStateTest, CircularListTest, TestRTGBase, CursorPagingTest, CdcrReplicationHandlerTest, CloudExitableDirectoryReaderTest, CollectionStateFormat2Test, CollectionTooManyReplicasTest, CollectionsAPISolrJTest, DeleteInactiveReplicaTest, DeleteNodeTest, DeleteStatusTest, DistributedQueueTest, ForceLeaderTest, LeaderElectionContextKeyTest, LeaderInitiatedRecoveryOnShardRestartTest, OverseerCollectionConfigSetProcessorTest, OverseerRolesTest, PeerSyncReplicationTest]
[junit4] Completed [501/677 (1!)] on J1 in 215.12s, 1 test, 1 failure <<< FAILURES!
[...truncated 62991 lines...]