You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/01/07 15:10:20 UTC
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3762
- Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3762/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
1 tests failed.
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
timeout waiting to see all nodes active
Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
at __randomizedtesting.SeedInfo.seed([633FC0C64BDC11BA:EB6BFF1CE5207C42]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:311)
at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:262)
at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:244)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 11359 lines...]
[junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/init-core-data-001
[junit4] 2> 1196766 INFO (SUITE-PeerSyncReplicationTest-seed#[633FC0C64BDC11BA]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 1196766 INFO (SUITE-PeerSyncReplicationTest-seed#[633FC0C64BDC11BA]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /n_b/zw
[junit4] 2> 1196768 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1196768 INFO (Thread-1598) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1196768 INFO (Thread-1598) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1196879 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.ZkTestServer start zk server on port:60387
[junit4] 2> 1196905 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1196924 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 1196927 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1196931 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1196934 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1196937 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1196940 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1196944 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1196947 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1196950 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1196952 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1198476 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/control-001/cores/collection1
[junit4] 2> 1198477 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1198480 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@66f92993{/n_b/zw,null,AVAILABLE}
[junit4] 2> 1198481 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@26998071{HTTP/1.1,[http/1.1]}{127.0.0.1:60390}
[junit4] 2> 1198481 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.Server Started @1207267ms
[junit4] 2> 1198481 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/tempDir-001/control/data, hostContext=/n_b/zw, hostPort=60390, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/control-001/cores}
[junit4] 2> 1198482 ERROR (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1198482 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1198482 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1198482 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1198482 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-07T14:04:29.351Z
[junit4] 2> 1198486 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1198486 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/control-001/solr.xml
[junit4] 2> 1198497 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60387/solr
[junit4] 2> 1198540 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60390_n_b%2Fzw ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60390_n_b%2Fzw
[junit4] 2> 1198542 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60390_n_b%2Fzw ] o.a.s.c.Overseer Overseer (id=97242177053655044-127.0.0.1:60390_n_b%2Fzw-n_0000000000) starting
[junit4] 2> 1198560 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60390_n_b%2Fzw ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60390_n_b%2Fzw
[junit4] 2> 1198563 INFO (zkCallback-1376-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1198605 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60390_n_b%2Fzw ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/control-001/cores
[junit4] 2> 1198606 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60390_n_b%2Fzw ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1198611 INFO (OverseerStateUpdate-97242177053655044-127.0.0.1:60390_n_b%2Fzw-n_0000000000) [n:127.0.0.1:60390_n_b%2Fzw ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1199639 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1199653 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1199754 WARN (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1199757 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1199781 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 1199781 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/control-001/cores/collection1/data/]
[junit4] 2> 1199781 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b0987d
[junit4] 2> 1199785 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=24.8125, floorSegmentMB=0.724609375, forceMergeDeletesPctAllowed=14.913952844207012, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 1199818 WARN (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1199832 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1199832 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1199837 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1199837 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1199837 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=41, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 1199838 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@d8902b8[collection1] main]
[junit4] 2> 1199840 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1199841 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1199841 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1199842 INFO (searcherExecutor-3626-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw x:collection1 c:control_collection) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@d8902b8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1199842 INFO (coreLoadExecutor-3625-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555874836078657536
[junit4] 2> 1199853 INFO (coreZkRegister-3618-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw x:collection1 c:control_collection) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1199853 INFO (coreZkRegister-3618-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw x:collection1 c:control_collection) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1199853 INFO (coreZkRegister-3618-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw x:collection1 c:control_collection) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60390/n_b/zw/collection1/
[junit4] 2> 1199853 INFO (coreZkRegister-3618-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw x:collection1 c:control_collection) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1199853 INFO (coreZkRegister-3618-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw x:collection1 c:control_collection) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:60390/n_b/zw/collection1/ has no replicas
[junit4] 2> 1199861 INFO (coreZkRegister-3618-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw x:collection1 c:control_collection) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60390/n_b/zw/collection1/ shard1
[junit4] 2> 1199962 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1199964 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60387/solr ready
[junit4] 2> 1199964 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1199964 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 1200025 INFO (coreZkRegister-3618-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw x:collection1 c:control_collection) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1200300 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-1-001/cores/collection1
[junit4] 2> 1200301 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-1-001
[junit4] 2> 1200302 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1200304 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@711835d5{/n_b/zw,null,AVAILABLE}
[junit4] 2> 1200305 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@64463e3c{HTTP/1.1,[http/1.1]}{127.0.0.1:60396}
[junit4] 2> 1200305 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.Server Started @1209091ms
[junit4] 2> 1200305 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/n_b/zw, hostPort=60396, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-1-001/cores}
[junit4] 2> 1200305 ERROR (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1200306 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1200306 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1200306 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1200306 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-07T14:04:31.175Z
[junit4] 2> 1200311 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1200311 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-1-001/solr.xml
[junit4] 2> 1200351 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60387/solr
[junit4] 2> 1200368 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1200378 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60396_n_b%2Fzw
[junit4] 2> 1200381 INFO (zkCallback-1380-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1200381 INFO (zkCallback-1376-thread-2-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1200383 INFO (zkCallback-1386-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1200423 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-1-001/cores
[junit4] 2> 1200423 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1200427 INFO (OverseerStateUpdate-97242177053655044-127.0.0.1:60390_n_b%2Fzw-n_0000000000) [n:127.0.0.1:60390_n_b%2Fzw ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1200540 INFO (zkCallback-1386-thread-2-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1201465 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1201495 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1201686 WARN (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1201691 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1201740 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1201741 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 1201741 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b0987d
[junit4] 2> 1201746 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=24.8125, floorSegmentMB=0.724609375, forceMergeDeletesPctAllowed=14.913952844207012, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 1201760 WARN (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1201785 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1201785 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1201787 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1201787 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1201789 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=41, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 1201800 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6dc70fc7[collection1] main]
[junit4] 2> 1201802 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1201803 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1201804 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1201805 INFO (searcherExecutor-3637-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw x:collection1 c:collection1) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6dc70fc7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1201806 INFO (coreLoadExecutor-3636-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555874838138060800
[junit4] 2> 1201820 INFO (coreZkRegister-3631-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw x:collection1 c:collection1) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1201820 INFO (coreZkRegister-3631-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw x:collection1 c:collection1) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1201820 INFO (coreZkRegister-3631-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw x:collection1 c:collection1) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60396/n_b/zw/collection1/
[junit4] 2> 1201820 INFO (coreZkRegister-3631-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw x:collection1 c:collection1) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1201820 INFO (coreZkRegister-3631-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw x:collection1 c:collection1) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:60396/n_b/zw/collection1/ has no replicas
[junit4] 2> 1201833 INFO (coreZkRegister-3631-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw x:collection1 c:collection1) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60396/n_b/zw/collection1/ shard1
[junit4] 2> 1201941 INFO (zkCallback-1386-thread-2-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1201988 INFO (coreZkRegister-3631-thread-1-processing-n:127.0.0.1:60396_n_b%2Fzw x:collection1 c:collection1) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1202096 INFO (zkCallback-1386-thread-2-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1202635 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-2-001/cores/collection1
[junit4] 2> 1202637 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-2-001
[junit4] 2> 1202638 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1202640 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6d1c623f{/n_b/zw,null,AVAILABLE}
[junit4] 2> 1202640 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2e3e523e{HTTP/1.1,[http/1.1]}{127.0.0.1:60400}
[junit4] 2> 1202640 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.Server Started @1211426ms
[junit4] 2> 1202640 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/n_b/zw, hostPort=60400, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-2-001/cores}
[junit4] 2> 1202641 ERROR (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1202641 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1202642 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1202642 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1202642 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-07T14:04:33.511Z
[junit4] 2> 1202645 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1202646 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-2-001/solr.xml
[junit4] 2> 1202663 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60387/solr
[junit4] 2> 1202681 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60400_n_b%2Fzw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1202690 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60400_n_b%2Fzw ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60400_n_b%2Fzw
[junit4] 2> 1202693 INFO (zkCallback-1376-thread-2-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1202694 INFO (zkCallback-1392-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1202693 INFO (zkCallback-1386-thread-2-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1202694 INFO (zkCallback-1380-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1202763 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60400_n_b%2Fzw ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-2-001/cores
[junit4] 2> 1202763 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60400_n_b%2Fzw ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1202767 INFO (OverseerStateUpdate-97242177053655044-127.0.0.1:60390_n_b%2Fzw-n_0000000000) [n:127.0.0.1:60390_n_b%2Fzw ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1202874 INFO (zkCallback-1386-thread-2-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1202874 INFO (zkCallback-1392-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1203792 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1203815 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1203939 WARN (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1203943 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1203970 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1203971 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 1203972 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b0987d
[junit4] 2> 1203977 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=24.8125, floorSegmentMB=0.724609375, forceMergeDeletesPctAllowed=14.913952844207012, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 1204011 WARN (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1204028 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1204028 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1204035 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1204035 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1204036 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=41, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 1204037 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1b066aba[collection1] main]
[junit4] 2> 1204039 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1204040 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1204040 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1204040 INFO (searcherExecutor-3648-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 c:collection1) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1b066aba[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1204041 INFO (coreLoadExecutor-3647-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555874840481628160
[junit4] 2> 1204047 INFO (coreZkRegister-3642-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 c:collection1) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 1204048 INFO (updateExecutor-1389-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 1204048 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 1204049 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 1204049 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 1204049 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 1204049 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:60396/n_b/zw/collection1/] and I am [http://127.0.0.1:60400/n_b/zw/collection1/]
[junit4] 2> 1204052 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:60396/n_b/zw]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:60400_n_b%252Fzw&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 1204056 INFO (qtp1726839415-7616) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 1204057 INFO (qtp1726839415-7616) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 1204057 INFO (qtp1726839415-7616) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:60400_n_b%2Fzw, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:60400/n_b/zw","node_name":"127.0.0.1:60400_n_b%2Fzw","state":"down"}
[junit4] 2> 1204160 INFO (zkCallback-1392-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1204160 INFO (zkCallback-1386-thread-2-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1204706 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-3-001/cores/collection1
[junit4] 2> 1204708 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-3-001
[junit4] 2> 1204708 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1204711 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4d36ad32{/n_b/zw,null,AVAILABLE}
[junit4] 2> 1204712 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@a4437d0{HTTP/1.1,[http/1.1]}{127.0.0.1:60406}
[junit4] 2> 1204712 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.Server Started @1213498ms
[junit4] 2> 1204712 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/n_b/zw, hostPort=60406, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-3-001/cores}
[junit4] 2> 1204713 ERROR (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1204714 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1204714 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1204714 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1204714 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-07T14:04:35.583Z
[junit4] 2> 1204721 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1204721 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-3-001/solr.xml
[junit4] 2> 1204754 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60387/solr
[junit4] 2> 1204778 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60406_n_b%2Fzw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1204791 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60406_n_b%2Fzw ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60406_n_b%2Fzw
[junit4] 2> 1204795 INFO (zkCallback-1392-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1204797 INFO (zkCallback-1399-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1204795 INFO (zkCallback-1386-thread-2-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1204795 INFO (zkCallback-1380-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1204795 INFO (zkCallback-1376-thread-1-processing-n:127.0.0.1:60390_n_b%2Fzw) [n:127.0.0.1:60390_n_b%2Fzw ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1204864 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60406_n_b%2Fzw ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-3-001/cores
[junit4] 2> 1204864 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [n:127.0.0.1:60406_n_b%2Fzw ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1204868 INFO (OverseerStateUpdate-97242177053655044-127.0.0.1:60390_n_b%2Fzw-n_0000000000) [n:127.0.0.1:60390_n_b%2Fzw ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1204974 INFO (zkCallback-1399-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1204974 INFO (zkCallback-1386-thread-2-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1204974 INFO (zkCallback-1392-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1205060 INFO (qtp1726839415-7616) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:60400_n_b%2Fzw, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:60400/n_b/zw","node_name":"127.0.0.1:60400_n_b%2Fzw","state":"recovering"}
[junit4] 2> 1205060 INFO (qtp1726839415-7616) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 1205060 INFO (qtp1726839415-7616) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:60400_n_b%252Fzw&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1003
[junit4] 2> 1205886 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1205901 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1205997 WARN (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1205999 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1206021 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1206022 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 1206022 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b0987d
[junit4] 2> 1206025 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=24.8125, floorSegmentMB=0.724609375, forceMergeDeletesPctAllowed=14.913952844207012, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 1206060 WARN (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1206077 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1206077 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1206078 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1206078 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1206079 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=41, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 1206079 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@408a4e90[collection1] main]
[junit4] 2> 1206081 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1206082 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1206082 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1206083 INFO (searcherExecutor-3659-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 c:collection1) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@408a4e90[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1206084 INFO (coreLoadExecutor-3658-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555874842622820352
[junit4] 2> 1206089 INFO (coreZkRegister-3653-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 c:collection1) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 1206090 INFO (updateExecutor-1396-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 1206090 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 1206091 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 1206091 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 1206091 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 1206091 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:60396/n_b/zw/collection1/] and I am [http://127.0.0.1:60406/n_b/zw/collection1/]
[junit4] 2> 1206093 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:60396/n_b/zw]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:60406_n_b%252Fzw&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 1206095 INFO (qtp1726839415-7620) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 1206096 INFO (qtp1726839415-7620) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 1206096 INFO (qtp1726839415-7620) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:60406_n_b%2Fzw, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:60406/n_b/zw","node_name":"127.0.0.1:60406_n_b%2Fzw","state":"down"}
[junit4] 2> 1206198 INFO (zkCallback-1399-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1206198 INFO (zkCallback-1386-thread-2-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1206198 INFO (zkCallback-1392-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1206343 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 1206343 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30 for each attempt
[junit4] 2> 1206343 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 1207096 INFO (qtp1726839415-7620) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:60406_n_b%2Fzw, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:60406/n_b/zw","node_name":"127.0.0.1:60406_n_b%2Fzw","state":"recovering"}
[junit4] 2> 1207097 INFO (qtp1726839415-7620) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 1207097 INFO (qtp1726839415-7620) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:60406_n_b%252Fzw&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1001
[junit4] 2> 1212068 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:60396/n_b/zw/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 1212068 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:60400/n_b/zw START replicas=[http://127.0.0.1:60396/n_b/zw/collection1/] nUpdates=1000
[junit4] 2> 1212072 INFO (qtp1726839415-7617) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:2.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1212073 INFO (qtp1726839415-7617) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp=/n_b/zw path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1212074 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1212074 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 1212074 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1212074 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1212075 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1212075 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 1212075 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 1212075 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 1212075 INFO (recoveryExecutor-1390-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 1212081 INFO (zkCallback-1386-thread-2-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1212081 INFO (zkCallback-1399-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1212081 INFO (zkCallback-1392-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1214097 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:60396/n_b/zw/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 1214098 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:60406/n_b/zw START replicas=[http://127.0.0.1:60396/n_b/zw/collection1/] nUpdates=1000
[junit4] 2> 1214101 INFO (qtp1726839415-7618) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1214101 INFO (qtp1726839415-7618) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp=/n_b/zw path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 1214102 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1214102 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 1214102 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1214102 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1214102 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1214103 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 1214103 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 1214103 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 1214103 INFO (recoveryExecutor-1397-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 1214107 INFO (zkCallback-1399-thread-1-processing-n:127.0.0.1:60406_n_b%2Fzw) [n:127.0.0.1:60406_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1214107 INFO (zkCallback-1386-thread-2-processing-n:127.0.0.1:60396_n_b%2Fzw) [n:127.0.0.1:60396_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1214108 INFO (zkCallback-1392-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1214364 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 1214369 INFO (qtp1041830688-7577) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1214369 INFO (qtp1041830688-7577) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1214370 INFO (qtp1041830688-7577) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1214371 INFO (qtp1041830688-7577) [n:127.0.0.1:60390_n_b%2Fzw c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/n_b/zw path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 1
[junit4] 2> 1214375 INFO (qtp1726839415-7614) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1214375 INFO (qtp1726839415-7614) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1214378 INFO (qtp1726839415-7614) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1214378 INFO (qtp1726839415-7614) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/n_b/zw path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:60396/n_b/zw/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 3
[junit4] 2> 1214379 INFO (qtp2087916659-7682) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1214379 INFO (qtp929874068-7647) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1214381 INFO (qtp2087916659-7682) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1214381 INFO (qtp929874068-7647) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1214381 INFO (qtp2087916659-7682) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1214381 INFO (qtp929874068-7647) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1214381 INFO (qtp2087916659-7682) [n:127.0.0.1:60406_n_b%2Fzw c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/n_b/zw path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:60396/n_b/zw/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 1214381 INFO (qtp929874068-7647) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/n_b/zw path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:60396/n_b/zw/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 1214381 INFO (qtp1726839415-7621) [n:127.0.0.1:60396_n_b%2Fzw c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/n_b/zw path=/update params={_stateVer_=collection1:9&waitSearcher=true&commit=true&softCommit=fal
[...truncated too long message...]
[junit4] 2> 1405825 INFO (recoveryExecutor-1404-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy RecoveryStrategy has been closed
[junit4] 2> 1405825 INFO (recoveryExecutor-1404-thread-1-processing-n:127.0.0.1:60400_n_b%2Fzw x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[false]
[junit4] 2> 1405826 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.Overseer Overseer (id=97242177053655058-127.0.0.1:60400_n_b%2Fzw-n_0000000004) closing
[junit4] 2> 1405826 INFO (OverseerStateUpdate-97242177053655058-127.0.0.1:60400_n_b%2Fzw-n_0000000004) [n:127.0.0.1:60400_n_b%2Fzw ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:60400_n_b%2Fzw
[junit4] 2> 1407327 INFO (zkCallback-1406-thread-2-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@ed98c83
[junit4] 2> 1407327 WARN (zkCallback-1406-thread-3-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1407328 WARN (zkCallback-1406-thread-2-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1] coreNodeName=[core_node2]
[junit4] 2> 1407345 INFO (zkCallback-1406-thread-2-processing-n:127.0.0.1:60400_n_b%2Fzw) [n:127.0.0.1:60400_n_b%2Fzw c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 1407345 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 1407346 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@215c86e5{HTTP/1.1,[http/1.1]}{127.0.0.1:60400}
[junit4] 2> 1407346 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@78e8c4b3{/n_b/zw,null,UNAVAILABLE}
[junit4] 2> 1407347 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 60406
[junit4] 2> 1407348 INFO (TEST-PeerSyncReplicationTest.test-seed#[633FC0C64BDC11BA]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60387 60387
[junit4] 2> 1407364 INFO (Thread-1598) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60387 60387
[junit4] 2> 1410135 WARN (Thread-1598) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2> 2 /solr/collections/collection1/leader_elect/shard1/election/97242177053655049-core_node1-n_0000000000
[junit4] 2> 2 /solr/overseer_elect/election/97242177053655049-127.0.0.1:60396_n_b%2Fzw-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 207 /solr/overseer/collection-queue-work
[junit4] 2> 38 /solr/overseer/queue
[junit4] 2> 14 /solr/overseer/queue-work
[junit4] 2> 6 /solr/collections
[junit4] 2> 5 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=PeerSyncReplicationTest -Dtests.method=test -Dtests.seed=633FC0C64BDC11BA -Dtests.slow=true -Dtests.locale=hi -Dtests.timezone=America/Goose_Bay -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 213s J1 | PeerSyncReplicationTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: timeout waiting to see all nodes active
[junit4] > at __randomizedtesting.SeedInfo.seed([633FC0C64BDC11BA:EB6BFF1CE5207C42]:0)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:311)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:262)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:244)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:133)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 1410139 INFO (SUITE-PeerSyncReplicationTest-seed#[633FC0C64BDC11BA]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_633FC0C64BDC11BA-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {other_tl1=FSTOrd50, range_facet_l_dv=PostingsFormat(name=Direct), rnd_s=FSTOrd50, multiDefault=BlockTreeOrds(blocksize=128), intDefault=FSTOrd50, a_i1=BlockTreeOrds(blocksize=128), range_facet_l=BlockTreeOrds(blocksize=128), _version_=FSTOrd50, a_t=BlockTreeOrds(blocksize=128), id=PostingsFormat(name=Direct), range_facet_i_dv=BlockTreeOrds(blocksize=128), text=FST50, timestamp=BlockTreeOrds(blocksize=128)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Asserting), range_facet_i_dv=DocValuesFormat(name=Lucene70), timestamp=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1611, maxMBSortInHeap=5.921736353052244, sim=RandomSimilarity(queryNorm=false): {}, locale=hi, timezone=America/Goose_Bay
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=3,threads=1,free=150505224,total=282763264
[junit4] 2> NOTE: All tests run in this JVM: [TestBadConfig, StressHdfsTest, ZkSolrClientTest, TestSolrCloudWithSecureImpersonation, ReplicationFactorTest, SuggesterTSTTest, TestSchemaManager, ZkStateReaderTest, ConjunctionSolrSpellCheckerTest, TestReloadDeadlock, TestSchemaVersionResource, BasicDistributedZk2Test, BasicAuthIntegrationTest, CreateCollectionCleanupTest, CloudMLTQParserTest, TestCrossCoreJoin, SolrCoreCheckLockOnStartupTest, ExitableDirectoryReaderTest, TestSSLRandomization, SolrJmxReporterTest, OverriddenZkACLAndCredentialsProvidersTest, TestOrdValues, TestReload, TestDeleteCollectionOnDownNodes, ReturnFieldsTest, TestSolrCLIRunExample, UpdateRequestProcessorFactoryTest, TestSolr4Spatial2, TestPivotHelperCode, TestFiltering, DirectUpdateHandlerOptimizeTest, SolrInfoMBeanTest, TestAuthenticationFramework, DefaultValueUpdateProcessorTest, DistributedFacetPivotSmallAdvancedTest, TestQueryTypes, TestSolrIndexConfig, TestDocSet, DeleteInactiveReplicaTest, WordBreakSolrSpellCheckerTest, TestReRankQParserPlugin, HdfsLockFactoryTest, SolrCmdDistributorTest, JSONWriterTest, TestImpersonationWithHadoopAuth, TestBinaryResponseWriter, SpellingQueryConverterTest, MinimalSchemaTest, TestRandomFlRTGCloud, TestFieldTypeResource, DistributedFacetPivotWhiteBoxTest, TestReversedWildcardFilterFactory, TestLeaderElectionZkExpiry, TestFieldCache, TestFreeTextSuggestions, TestChildDocTransformer, ResponseLogComponentTest, TestLMDirichletSimilarityFactory, TestCloudPseudoReturnFields, CSVRequestHandlerTest, BadIndexSchemaTest, HdfsRecoverLeaseTest, HighlighterConfigTest, EchoParamsTest, TestFastWriter, TestLocalFSCloudBackupRestore, TestRandomCollapseQParserPlugin, TestExceedMaxTermLength, TestUseDocValuesAsStored2, DistributedIntervalFacetingTest, BadComponentTest, TestConfigSetsAPIZkFailure, PreAnalyzedFieldManagedSchemaCloudTest, TestSystemIdResolver, ShardRoutingCustomTest, UUIDFieldTest, LeaderElectionContextKeyTest, TestAnalyzedSuggestions, OverseerModifyCollectionTest, TestJsonFacets, BigEndianAscendingWordSerializerTest, TestRandomFaceting, ExternalFileFieldSortTest, TestShortCircuitedRequests, TestXmlQParserPlugin, CdcrVersionReplicationTest, TestIndexSearcher, TestLuceneMatchVersion, DateMathParserTest, TestPushWriter, ClusterStateTest, ResourceLoaderTest, HdfsChaosMonkeyNothingIsSafeTest, SpellCheckCollatorTest, TestUnifiedSolrHighlighter, TestSolrCloudWithDelegationTokens, CollectionReloadTest, NoCacheHeaderTest, TestLMJelinekMercerSimilarityFactory, TestSQLHandlerNonCloud, BasicZkTest, CoreSorterTest, TestCollationFieldDocValues, LeaderElectionIntegrationTest, TermVectorComponentDistributedTest, TestConfigReload, RegexBoostProcessorTest, DistribJoinFromCollectionTest, SuggesterWFSTTest, TestQueryUtils, DeleteNodeTest, TestManagedSchemaAPI, IndexSchemaTest, AutoCommitTest, TestRealTimeGet, TestSolrDynamicMBean, CollectionsAPIDistributedZkTest, CollectionsAPISolrJTest, FacetPivotSmallTest, SharedFSAutoReplicaFailoverTest, TestTolerantUpdateProcessorRandomCloud, TestSearchPerf, TestPHPSerializedResponseWriter, TestShardHandlerFactory, LukeRequestHandlerTest, PeerSyncReplicationTest]
[junit4] Completed [224/675 (1!)] on J1 in 213.40s, 1 test, 1 failure <<< FAILURES!
[...truncated 63888 lines...]
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3764
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3764/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC
1 tests failed.
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
timeout waiting to see all nodes active
Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
at __randomizedtesting.SeedInfo.seed([33167BFC955BE930:BB4244263BA784C8]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:311)
at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:262)
at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:244)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 10874 lines...]
[junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/init-core-data-001
[junit4] 2> 186155 INFO (SUITE-PeerSyncReplicationTest-seed#[33167BFC955BE930]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 186155 INFO (SUITE-PeerSyncReplicationTest-seed#[33167BFC955BE930]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 186163 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 186164 INFO (Thread-250) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 186164 INFO (Thread-250) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 186269 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.ZkTestServer start zk server on port:53672
[junit4] 2> 186310 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x1597db676480000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 186352 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 186359 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 186364 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 186368 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 186372 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 186377 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 186380 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 186385 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 186396 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 186401 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 186404 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 186949 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/control-001/cores/collection1
[junit4] 2> 186952 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 186955 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@75da8018{/,null,AVAILABLE}
[junit4] 2> 186961 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6d4ec9b9{HTTP/1.1,[http/1.1]}{127.0.0.1:53675}
[junit4] 2> 186961 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.Server Started @193077ms
[junit4] 2> 186962 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/tempDir-001/control/data, hostContext=/, hostPort=53675, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/control-001/cores}
[junit4] 2> 186962 ERROR (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 186962 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 186962 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 186962 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 186962 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-08T10:53:47.745Z
[junit4] 2> 186976 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 186976 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/control-001/solr.xml
[junit4] 2> 187011 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53672/solr
[junit4] 2> 187072 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53675_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:53675_
[junit4] 2> 187075 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53675_ ] o.a.s.c.Overseer Overseer (id=97247089587126276-127.0.0.1:53675_-n_0000000000) starting
[junit4] 2> 187099 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53675_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53675_
[junit4] 2> 187103 INFO (zkCallback-138-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 187257 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53675_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/control-001/cores
[junit4] 2> 187257 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53675_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 187275 INFO (OverseerStateUpdate-97247089587126276-127.0.0.1:53675_-n_0000000000) [n:127.0.0.1:53675_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 188302 WARN (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 188303 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 188338 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 188531 WARN (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 188540 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 188591 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 188616 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/control-001/cores/collection1/data/]
[junit4] 2> 188616 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7683e2fc
[junit4] 2> 188624 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=75.375, floorSegmentMB=1.296875, forceMergeDeletesPctAllowed=2.68641818121097, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8280606795252152
[junit4] 2> 188640 WARN (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 188665 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 188666 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 188667 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 188667 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 188668 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.34896130290445476]
[junit4] 2> 188685 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7dc56a18[collection1] main]
[junit4] 2> 188694 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 188694 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 188695 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 188696 INFO (searcherExecutor-533-thread-1-processing-n:127.0.0.1:53675_ x:collection1 c:control_collection) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7dc56a18[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 188697 INFO (coreLoadExecutor-532-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555953436044820480
[junit4] 2> 188711 INFO (coreZkRegister-525-thread-1-processing-n:127.0.0.1:53675_ x:collection1 c:control_collection) [n:127.0.0.1:53675_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 188712 INFO (coreZkRegister-525-thread-1-processing-n:127.0.0.1:53675_ x:collection1 c:control_collection) [n:127.0.0.1:53675_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 188712 INFO (coreZkRegister-525-thread-1-processing-n:127.0.0.1:53675_ x:collection1 c:control_collection) [n:127.0.0.1:53675_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53675/collection1/
[junit4] 2> 188712 INFO (coreZkRegister-525-thread-1-processing-n:127.0.0.1:53675_ x:collection1 c:control_collection) [n:127.0.0.1:53675_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 188712 INFO (coreZkRegister-525-thread-1-processing-n:127.0.0.1:53675_ x:collection1 c:control_collection) [n:127.0.0.1:53675_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:53675/collection1/ has no replicas
[junit4] 2> 188722 INFO (coreZkRegister-525-thread-1-processing-n:127.0.0.1:53675_ x:collection1 c:control_collection) [n:127.0.0.1:53675_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53675/collection1/ shard1
[junit4] 2> 188881 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 188884 INFO (coreZkRegister-525-thread-1-processing-n:127.0.0.1:53675_ x:collection1 c:control_collection) [n:127.0.0.1:53675_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 188884 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53672/solr ready
[junit4] 2> 188884 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 189405 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-1-001/cores/collection1
[junit4] 2> 189407 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-1-001
[junit4] 2> 189408 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 189410 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@425ca3d7{/,null,AVAILABLE}
[junit4] 2> 189411 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2ff9b1d0{HTTP/1.1,[http/1.1]}{127.0.0.1:53682}
[junit4] 2> 189411 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.Server Started @195526ms
[junit4] 2> 189411 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=53682, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-1-001/cores}
[junit4] 2> 189412 ERROR (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 189413 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 189413 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 189413 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 189413 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-08T10:53:50.196Z
[junit4] 2> 189421 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 189422 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-1-001/solr.xml
[junit4] 2> 189476 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53672/solr
[junit4] 2> 189509 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53682_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 189535 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53682_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53682_
[junit4] 2> 189541 INFO (zkCallback-142-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 189541 INFO (zkCallback-138-thread-2-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 189543 INFO (zkCallback-147-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 189654 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53682_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-1-001/cores
[junit4] 2> 189654 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53682_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 189660 INFO (OverseerStateUpdate-97247089587126276-127.0.0.1:53675_-n_0000000000) [n:127.0.0.1:53675_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 190689 WARN (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 190690 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 190726 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 190877 WARN (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 190881 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 190915 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 190915 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 190915 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7683e2fc
[junit4] 2> 190921 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=75.375, floorSegmentMB=1.296875, forceMergeDeletesPctAllowed=2.68641818121097, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8280606795252152
[junit4] 2> 190940 WARN (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 190968 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 190968 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 190969 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 190970 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 190971 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.34896130290445476]
[junit4] 2> 190987 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7171498f[collection1] main]
[junit4] 2> 190989 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 190989 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 190990 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 190991 INFO (searcherExecutor-544-thread-1-processing-n:127.0.0.1:53682_ x:collection1 c:collection1) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7171498f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 190992 INFO (coreLoadExecutor-543-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555953438451302400
[junit4] 2> 191006 INFO (coreZkRegister-538-thread-1-processing-n:127.0.0.1:53682_ x:collection1 c:collection1) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 191006 INFO (coreZkRegister-538-thread-1-processing-n:127.0.0.1:53682_ x:collection1 c:collection1) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 191006 INFO (coreZkRegister-538-thread-1-processing-n:127.0.0.1:53682_ x:collection1 c:collection1) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53682/collection1/
[junit4] 2> 191006 INFO (coreZkRegister-538-thread-1-processing-n:127.0.0.1:53682_ x:collection1 c:collection1) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 191006 INFO (coreZkRegister-538-thread-1-processing-n:127.0.0.1:53682_ x:collection1 c:collection1) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:53682/collection1/ has no replicas
[junit4] 2> 191015 INFO (coreZkRegister-538-thread-1-processing-n:127.0.0.1:53682_ x:collection1 c:collection1) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53682/collection1/ shard1
[junit4] 2> 191171 INFO (coreZkRegister-538-thread-1-processing-n:127.0.0.1:53682_ x:collection1 c:collection1) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 192136 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-2-001/cores/collection1
[junit4] 2> 192138 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-2-001
[junit4] 2> 192139 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 192142 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@937098a{/,null,AVAILABLE}
[junit4] 2> 192142 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@417188c{HTTP/1.1,[http/1.1]}{127.0.0.1:53699}
[junit4] 2> 192143 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.Server Started @198259ms
[junit4] 2> 192143 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=53699, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-2-001/cores}
[junit4] 2> 192144 ERROR (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 192145 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 192145 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 192145 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 192145 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-08T10:53:52.928Z
[junit4] 2> 192153 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 192153 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-2-001/solr.xml
[junit4] 2> 192184 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53672/solr
[junit4] 2> 192211 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53699_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 192225 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53699_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53699_
[junit4] 2> 192228 INFO (zkCallback-147-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 192230 INFO (zkCallback-138-thread-1-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 192231 INFO (zkCallback-153-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 192230 INFO (zkCallback-142-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 192338 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53699_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-2-001/cores
[junit4] 2> 192338 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53699_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 192352 INFO (OverseerStateUpdate-97247089587126276-127.0.0.1:53675_-n_0000000000) [n:127.0.0.1:53675_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 193366 WARN (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 193367 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 193443 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 193607 WARN (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 193610 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 193640 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 193641 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 193641 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7683e2fc
[junit4] 2> 193651 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=75.375, floorSegmentMB=1.296875, forceMergeDeletesPctAllowed=2.68641818121097, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8280606795252152
[junit4] 2> 193666 WARN (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 193690 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 193690 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 193691 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 193691 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 193692 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.34896130290445476]
[junit4] 2> 193705 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2c830ed5[collection1] main]
[junit4] 2> 193708 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 193713 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 193713 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 193714 INFO (searcherExecutor-555-thread-1-processing-n:127.0.0.1:53699_ x:collection1 c:collection1) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2c830ed5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 193715 INFO (coreLoadExecutor-554-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555953441306574848
[junit4] 2> 193721 INFO (coreZkRegister-549-thread-1-processing-n:127.0.0.1:53699_ x:collection1 c:collection1) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 193721 INFO (updateExecutor-150-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 193722 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 193722 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 193722 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 193722 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 193722 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:53682/collection1/] and I am [http://127.0.0.1:53699/collection1/]
[junit4] 2> 193725 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:53682]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:53699_&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 193730 INFO (qtp174779393-1041) [n:127.0.0.1:53682_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 193731 INFO (qtp174779393-1041) [n:127.0.0.1:53682_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 193731 INFO (qtp174779393-1041) [n:127.0.0.1:53682_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:53699_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:53699","node_name":"127.0.0.1:53699_","state":"down"}
[junit4] 2> 194298 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-3-001/cores/collection1
[junit4] 2> 194300 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-3-001
[junit4] 2> 194300 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 194303 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@710090a9{/,null,AVAILABLE}
[junit4] 2> 194304 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2aa0fdd3{HTTP/1.1,[http/1.1]}{127.0.0.1:53704}
[junit4] 2> 194304 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.Server Started @200420ms
[junit4] 2> 194304 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=53704, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-3-001/cores}
[junit4] 2> 194304 ERROR (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 194305 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 194305 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 194305 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 194305 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-08T10:53:55.088Z
[junit4] 2> 194313 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 194313 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-3-001/solr.xml
[junit4] 2> 194430 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53672/solr
[junit4] 2> 194474 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53704_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 194503 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53704_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53704_
[junit4] 2> 194511 INFO (zkCallback-138-thread-2-processing-n:127.0.0.1:53675_) [n:127.0.0.1:53675_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 194512 INFO (zkCallback-160-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 194513 INFO (zkCallback-153-thread-1-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 194513 INFO (zkCallback-147-thread-1-processing-n:127.0.0.1:53682_) [n:127.0.0.1:53682_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 194513 INFO (zkCallback-142-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 194618 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53704_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-3-001/cores
[junit4] 2> 194618 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [n:127.0.0.1:53704_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 194628 INFO (OverseerStateUpdate-97247089587126276-127.0.0.1:53675_-n_0000000000) [n:127.0.0.1:53675_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 194733 INFO (qtp174779393-1041) [n:127.0.0.1:53682_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:53699_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:53699","node_name":"127.0.0.1:53699_","state":"recovering"}
[junit4] 2> 194733 INFO (qtp174779393-1041) [n:127.0.0.1:53682_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 194734 INFO (qtp174779393-1041) [n:127.0.0.1:53682_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:53699_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1003
[junit4] 2> 195645 WARN (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 195645 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 195660 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 195767 WARN (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 195769 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 195792 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 195792 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 195792 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7683e2fc
[junit4] 2> 195795 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=75.375, floorSegmentMB=1.296875, forceMergeDeletesPctAllowed=2.68641818121097, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8280606795252152
[junit4] 2> 195806 WARN (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 195826 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 195826 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 195828 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 195828 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 195829 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.34896130290445476]
[junit4] 2> 195851 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3719f193[collection1] main]
[junit4] 2> 195853 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 195853 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 195853 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 195854 INFO (searcherExecutor-566-thread-1-processing-n:127.0.0.1:53704_ x:collection1 c:collection1) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3719f193[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 195855 INFO (coreLoadExecutor-565-thread-1-processing-n:127.0.0.1:53704_) [n:127.0.0.1:53704_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555953443550527488
[junit4] 2> 195861 INFO (coreZkRegister-560-thread-1-processing-n:127.0.0.1:53704_ x:collection1 c:collection1) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 195861 INFO (updateExecutor-157-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 195861 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 195862 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 195862 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 195862 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 195862 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:53682/collection1/] and I am [http://127.0.0.1:53704/collection1/]
[junit4] 2> 195865 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:53682]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:53704_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 195866 INFO (qtp174779393-1045) [n:127.0.0.1:53682_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 195867 INFO (qtp174779393-1045) [n:127.0.0.1:53682_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 195867 INFO (qtp174779393-1045) [n:127.0.0.1:53682_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:53704_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:53704","node_name":"127.0.0.1:53704_","state":"down"}
[junit4] 2> 196220 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 196220 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30 for each attempt
[junit4] 2> 196220 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 196871 INFO (qtp174779393-1045) [n:127.0.0.1:53682_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:53704_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:53704","node_name":"127.0.0.1:53704_","state":"recovering"}
[junit4] 2> 196871 INFO (qtp174779393-1045) [n:127.0.0.1:53682_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 196871 INFO (qtp174779393-1045) [n:127.0.0.1:53682_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:53704_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1005
[junit4] 2> 201743 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:53682/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 201743 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:53699 START replicas=[http://127.0.0.1:53682/collection1/] nUpdates=1000
[junit4] 2> 201747 INFO (qtp174779393-1042) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 201747 INFO (qtp174779393-1042) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 201749 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 201749 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 201749 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 201749 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 201749 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 201750 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 201750 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 201750 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 201750 INFO (recoveryExecutor-151-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 203874 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:53682/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 203884 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:53704 START replicas=[http://127.0.0.1:53682/collection1/] nUpdates=1000
[junit4] 2> 203887 INFO (qtp174779393-1043) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 203887 INFO (qtp174779393-1043) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 203888 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 203888 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 203889 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 203889 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 203890 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 203890 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 203890 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 203890 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 203890 INFO (recoveryExecutor-158-thread-1-processing-n:127.0.0.1:53704_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 204248 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 204252 INFO (qtp1731993310-1004) [n:127.0.0.1:53675_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 204253 INFO (qtp1731993310-1004) [n:127.0.0.1:53675_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 204254 INFO (qtp1731993310-1004) [n:127.0.0.1:53675_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 204254 INFO (qtp1731993310-1004) [n:127.0.0.1:53675_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 2
[junit4] 2> 204259 INFO (qtp174779393-1041) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 204259 INFO (qtp174779393-1041) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 204262 INFO (qtp174779393-1041) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 204262 INFO (qtp1552328449-1105) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 204263 INFO (qtp1552328449-1105) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 204263 INFO (qtp174779393-1041) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53682/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 4
[junit4] 2> 204264 INFO (qtp1552328449-1105) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 204264 INFO (qtp1552328449-1105) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53682/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 204264 INFO (qtp928430656-1069) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 204265 INFO (qtp928430656-1069) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 204265 INFO (qtp928430656-1069) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 204265 INFO (qtp928430656-1069) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53682/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 204266 INFO (qtp174779393-1046) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 9
[junit4] 2> 204268 INFO (qtp174779393-1042) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 204270 INFO (qtp928430656-1070) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 204273 INFO (qtp1552328449-1107) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 206280 INFO (qtp1731993310-1005) [n:127.0.0.1:53675_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1555953454477737984)} 0 4
[junit4] 2> 206289 INFO (qtp1552328449-1108) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1555953454482980864&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1555953454482980864)} 0 4
[junit4] 2> 206289 INFO (qtp928430656-1071) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1555953454482980864&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1555953454482980864)} 0 4
[junit4] 2> 206290 INFO (qtp174779393-1044) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1555953454482980864)} 0 9
[junit4] 2> 206301 INFO (qtp928430656-1072) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[0 (1555953454496612352)]} 0 3
[junit4] 2> 206301 INFO (qtp1552328449-1109) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[0 (1555953454496612352)]} 0 3
[junit4] 2> 206304 INFO (qtp174779393-1043) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[0 (1555953454496612352)]} 0 10
[junit4] 2> 206309 INFO (qtp928430656-1073) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[1 (1555953454508146688)]} 0 1
[junit4] 2> 206310 INFO (qtp1552328449-1110) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[1 (1555953454508146688)]} 0 2
[junit4] 2> 206310 INFO (qtp174779393-1045) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[1 (1555953454508146688)]} 0 5
[junit4] 2> 206313 INFO (qtp928430656-1074) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[2 (1555953454514438144)]} 0 0
[junit4] 2> 206313 INFO (qtp1552328449-1103) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[2 (1555953454514438144)]} 0 0
[junit4] 2> 206314 INFO (qtp174779393-1041) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[2 (1555953454514438144)]} 0 2
[junit4] 2> 206316 INFO (qtp928430656-1074) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[3 (1555953454518632448)]} 0 0
[junit4] 2> 206316 INFO (qtp1552328449-1103) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[3 (1555953454518632448)]} 0 0
[junit4] 2> 206317 INFO (qtp174779393-1046) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[3 (1555953454518632448)]} 0 2
[junit4] 2> 206319 INFO (qtp928430656-1074) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[4 (1555953454521778176)]} 0 0
[junit4] 2> 206320 INFO (qtp1552328449-1103) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[4 (1555953454521778176)]} 0 0
[junit4] 2> 206320 INFO (qtp174779393-1039) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[4 (1555953454521778176)]} 0 2
[junit4] 2> 206322 INFO (qtp928430656-1074) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[5 (1555953454524923904)]} 0 0
[junit4] 2> 206322 INFO (qtp1552328449-1107) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[5 (1555953454524923904)]} 0 0
[junit4] 2> 206323 INFO (qtp174779393-1042) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[5 (1555953454524923904)]} 0 1
[junit4] 2> 206325 INFO (qtp1552328449-1108) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[6 (1555953454527021056)]} 0 0
[junit4] 2> 206325 INFO (qtp928430656-1071) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[6 (1555953454527021056)]} 0 0
[junit4] 2> 206325 INFO (qtp174779393-1044) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[6 (1555953454527021056)]} 0 1
[junit4] 2> 206327 INFO (qtp928430656-1072) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[7 (1555953454530166784)]} 0 0
[junit4] 2> 206327 INFO (qtp1552328449-1108) [n:127.0.0.1:53704_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:53682/collection1/&wt=javabin&version=2}{add=[7 (1555953454530166784)]} 0 0
[junit4] 2> 206328 INFO (qtp174779393-1043) [n:127.0.0.1:53682_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&ve
[...truncated too long message...]
rg.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 392712 INFO (qtp174779393-1218) [n:127.0.0.1:53682_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:53699_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=400 QTime=33232
[junit4] 2> 392722 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 53699
[junit4] 2> 392722 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1021390012
[junit4] 2> 392722 WARN (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1] coreNodeName=[core_node2]
[junit4] 2> 392726 WARN (updateExecutor-164-thread-2-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Skipping recovery because Solr is shutdown
[junit4] 2> 396976 INFO (recoveryExecutor-165-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy RecoveryStrategy has been closed
[junit4] 2> 396976 INFO (recoveryExecutor-165-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[false]
[junit4] 2> 396976 INFO (recoveryExecutor-165-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@27b43434
[junit4] 2> 396977 WARN (recoveryExecutor-165-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1] coreNodeName=[core_node2]
[junit4] 2> 396999 INFO (recoveryExecutor-165-thread-1-processing-n:127.0.0.1:53699_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53699_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 397000 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.Overseer Overseer (id=97247089587126289-127.0.0.1:53699_-n_0000000004) closing
[junit4] 2> 397000 INFO (OverseerStateUpdate-97247089587126289-127.0.0.1:53699_-n_0000000004) [n:127.0.0.1:53699_ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:53699_
[junit4] 2> 397003 WARN (zkCallback-167-thread-3-processing-n:127.0.0.1:53699_) [n:127.0.0.1:53699_ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 397003 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 397004 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@34fa2005{HTTP/1.1,[http/1.1]}{127.0.0.1:53699}
[junit4] 2> 397005 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6dde8953{/,null,UNAVAILABLE}
[junit4] 2> 397010 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 53704
[junit4] 2> 397011 INFO (TEST-PeerSyncReplicationTest.test-seed#[33167BFC955BE930]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53672 53672
[junit4] 2> 397034 INFO (Thread-250) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53672 53672
[junit4] 2> 398746 WARN (Thread-250) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 2 /solr/overseer_elect/election/97247089587126280-127.0.0.1:53682_-n_0000000001
[junit4] 2> 2 /solr/collections/collection1/leader_elect/shard1/election/97247089587126280-core_node1-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 209 /solr/overseer/collection-queue-work
[junit4] 2> 38 /solr/overseer/queue
[junit4] 2> 6 /solr/collections
[junit4] 2> 6 /solr/overseer/queue-work
[junit4] 2> 5 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=PeerSyncReplicationTest -Dtests.method=test -Dtests.seed=33167BFC955BE930 -Dtests.slow=true -Dtests.locale=fr-CA -Dtests.timezone=Africa/Monrovia -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 213s J1 | PeerSyncReplicationTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: timeout waiting to see all nodes active
[junit4] > at __randomizedtesting.SeedInfo.seed([33167BFC955BE930:BB4244263BA784C8]:0)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:311)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:262)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:244)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:133)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 398755 INFO (SUITE-PeerSyncReplicationTest-seed#[33167BFC955BE930]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_33167BFC955BE930-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {other_tl1=PostingsFormat(name=Asserting), range_facet_l_dv=PostingsFormat(name=Memory), rnd_s=PostingsFormat(name=Asserting), multiDefault=PostingsFormat(name=MockRandom), intDefault=PostingsFormat(name=Asserting), a_i1=PostingsFormat(name=MockRandom), range_facet_l=PostingsFormat(name=MockRandom), _version_=PostingsFormat(name=Asserting), a_t=PostingsFormat(name=MockRandom), id=PostingsFormat(name=Memory), range_facet_i_dv=PostingsFormat(name=MockRandom), text=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), timestamp=PostingsFormat(name=MockRandom)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene70), range_facet_i_dv=DocValuesFormat(name=Direct), timestamp=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=261, maxMBSortInHeap=7.182877028615015, sim=RandomSimilarity(queryNorm=false): {}, locale=fr-CA, timezone=Africa/Monrovia
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=3,threads=1,free=62364320,total=239771648
[junit4] 2> NOTE: All tests run in this JVM: [TestCodecSupport, SolrCmdDistributorTest, FieldAnalysisRequestHandlerTest, TestJmxMonitoredMap, OverseerModifyCollectionTest, UpdateParamsTest, TestDocTermOrds, DistributedIntervalFacetingTest, TestPerFieldSimilarityWithDefaultOverride, FullSolrCloudDistribCmdsTest, InfoHandlerTest, TestSolrIndexConfig, PeerSyncReplicationTest]
[junit4] Completed [66/676 (1!)] on J1 in 212.64s, 1 test, 1 failure <<< FAILURES!
[...truncated 64392 lines...]
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3763
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3763/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
1 tests failed.
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
timeout waiting to see all nodes active
Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
at __randomizedtesting.SeedInfo.seed([46C8FCAE122A307B:CE9CC374BCD65D83]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:311)
at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:262)
at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:244)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:133)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12285 lines...]
[junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/init-core-data-001
[junit4] 2> 2521164 INFO (SUITE-PeerSyncReplicationTest-seed#[46C8FCAE122A307B]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 2521164 INFO (SUITE-PeerSyncReplicationTest-seed#[46C8FCAE122A307B]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /sovt/cq
[junit4] 2> 2521166 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2521167 INFO (Thread-3220) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2521167 INFO (Thread-3220) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2521271 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.ZkTestServer start zk server on port:54240
[junit4] 2> 2521301 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2521366 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 2521482 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2521510 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2521566 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2521582 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2521607 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2521644 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2521669 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2521697 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2521715 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2526606 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/control-001/cores/collection1
[junit4] 2> 2526608 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2526610 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5b83913f{/sovt/cq,null,AVAILABLE}
[junit4] 2> 2526610 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@788a31a7{HTTP/1.1,[http/1.1]}{127.0.0.1:54248}
[junit4] 2> 2526611 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.Server Started @2533201ms
[junit4] 2> 2526611 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/tempDir-001/control/data, hostContext=/sovt/cq, hostPort=54248, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/control-001/cores}
[junit4] 2> 2526611 ERROR (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2526611 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2526611 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2526612 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2526612 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-08T00:35:55.665Z
[junit4] 2> 2526615 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2526615 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/control-001/solr.xml
[junit4] 2> 2526645 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54240/solr
[junit4] 2> 2526687 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54248_sovt%2Fcq ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:54248_sovt%2Fcq
[junit4] 2> 2526689 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54248_sovt%2Fcq ] o.a.s.c.Overseer Overseer (id=97244659726548996-127.0.0.1:54248_sovt%2Fcq-n_0000000000) starting
[junit4] 2> 2526706 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54248_sovt%2Fcq ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54248_sovt%2Fcq
[junit4] 2> 2526711 INFO (zkCallback-3387-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2527145 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54248_sovt%2Fcq ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/control-001/cores
[junit4] 2> 2527145 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54248_sovt%2Fcq ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2527150 INFO (OverseerStateUpdate-97244659726548996-127.0.0.1:54248_sovt%2Fcq-n_0000000000) [n:127.0.0.1:54248_sovt%2Fcq ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2528165 WARN (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2528166 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2528180 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2528267 WARN (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2528270 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2528291 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 2528292 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/control-001/cores/collection1/data/]
[junit4] 2> 2528292 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7ebb4ac1
[junit4] 2> 2528296 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=11, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=78.751953125, floorSegmentMB=1.8701171875, forceMergeDeletesPctAllowed=26.71723740692539, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6695027851037475
[junit4] 2> 2528304 WARN (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2528316 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2528316 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2528317 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2528317 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2528317 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1097877596, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2528318 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6af903db[collection1] main]
[junit4] 2> 2528320 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2528320 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2528320 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2528321 INFO (searcherExecutor-8197-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq x:collection1 c:control_collection) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6af903db[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2528321 INFO (coreLoadExecutor-8196-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555914563124199424
[junit4] 2> 2528333 INFO (coreZkRegister-8189-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq x:collection1 c:control_collection) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2528333 INFO (coreZkRegister-8189-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq x:collection1 c:control_collection) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2528333 INFO (coreZkRegister-8189-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq x:collection1 c:control_collection) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:54248/sovt/cq/collection1/
[junit4] 2> 2528333 INFO (coreZkRegister-8189-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq x:collection1 c:control_collection) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2528333 INFO (coreZkRegister-8189-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq x:collection1 c:control_collection) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:54248/sovt/cq/collection1/ has no replicas
[junit4] 2> 2528341 INFO (coreZkRegister-8189-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq x:collection1 c:control_collection) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:54248/sovt/cq/collection1/ shard1
[junit4] 2> 2528494 INFO (coreZkRegister-8189-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq x:collection1 c:control_collection) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2528576 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2528578 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:54240/solr ready
[junit4] 2> 2528578 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 2529078 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-1-001/cores/collection1
[junit4] 2> 2529089 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-1-001
[junit4] 2> 2529089 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2529092 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3ff83609{/sovt/cq,null,AVAILABLE}
[junit4] 2> 2529093 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@23f69f0e{HTTP/1.1,[http/1.1]}{127.0.0.1:54258}
[junit4] 2> 2529093 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.Server Started @2535683ms
[junit4] 2> 2529093 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/sovt/cq, hostPort=54258, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-1-001/cores}
[junit4] 2> 2529094 ERROR (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2529094 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2529094 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2529094 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2529094 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-08T00:35:58.147Z
[junit4] 2> 2529098 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2529098 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-1-001/solr.xml
[junit4] 2> 2529162 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54240/solr
[junit4] 2> 2529187 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2529196 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54258_sovt%2Fcq
[junit4] 2> 2529199 INFO (zkCallback-3391-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2529199 INFO (zkCallback-3387-thread-2-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2529200 INFO (zkCallback-3396-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2529244 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-1-001/cores
[junit4] 2> 2529244 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2529248 INFO (OverseerStateUpdate-97244659726548996-127.0.0.1:54248_sovt%2Fcq-n_0000000000) [n:127.0.0.1:54248_sovt%2Fcq ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2530273 WARN (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2530274 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2530284 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2530374 WARN (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2530377 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2530396 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2530396 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 2530396 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7ebb4ac1
[junit4] 2> 2530400 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=11, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=78.751953125, floorSegmentMB=1.8701171875, forceMergeDeletesPctAllowed=26.71723740692539, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6695027851037475
[junit4] 2> 2530463 WARN (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2530475 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2530475 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2530476 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2530476 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2530478 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1097877596, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2530479 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@558b84a5[collection1] main]
[junit4] 2> 2530480 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2530481 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2530481 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2530482 INFO (searcherExecutor-8208-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq x:collection1 c:collection1) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@558b84a5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2530482 INFO (coreLoadExecutor-8207-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555914565390172160
[junit4] 2> 2530492 INFO (coreZkRegister-8202-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq x:collection1 c:collection1) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2530492 INFO (coreZkRegister-8202-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq x:collection1 c:collection1) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2530492 INFO (coreZkRegister-8202-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq x:collection1 c:collection1) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:54258/sovt/cq/collection1/
[junit4] 2> 2530492 INFO (coreZkRegister-8202-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq x:collection1 c:collection1) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2530492 INFO (coreZkRegister-8202-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq x:collection1 c:collection1) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:54258/sovt/cq/collection1/ has no replicas
[junit4] 2> 2530499 INFO (coreZkRegister-8202-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq x:collection1 c:collection1) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:54258/sovt/cq/collection1/ shard1
[junit4] 2> 2530655 INFO (coreZkRegister-8202-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq x:collection1 c:collection1) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2531149 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-2-001/cores/collection1
[junit4] 2> 2531150 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-2-001
[junit4] 2> 2531150 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2531152 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@30ab96e3{/sovt/cq,null,AVAILABLE}
[junit4] 2> 2531152 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@38c64367{HTTP/1.1,[http/1.1]}{127.0.0.1:54262}
[junit4] 2> 2531152 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.Server Started @2537743ms
[junit4] 2> 2531153 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/sovt/cq, hostPort=54262, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-2-001/cores}
[junit4] 2> 2531153 ERROR (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2531154 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2531154 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2531154 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2531154 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-08T00:36:00.207Z
[junit4] 2> 2531158 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2531158 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-2-001/solr.xml
[junit4] 2> 2531197 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54240/solr
[junit4] 2> 2531216 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54262_sovt%2Fcq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2531225 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54262_sovt%2Fcq ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54262_sovt%2Fcq
[junit4] 2> 2531228 INFO (zkCallback-3396-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2531228 INFO (zkCallback-3387-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2531228 INFO (zkCallback-3391-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2531229 INFO (zkCallback-3402-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2531304 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54262_sovt%2Fcq ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-2-001/cores
[junit4] 2> 2531305 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54262_sovt%2Fcq ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2531307 INFO (OverseerStateUpdate-97244659726548996-127.0.0.1:54248_sovt%2Fcq-n_0000000000) [n:127.0.0.1:54248_sovt%2Fcq ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2532322 WARN (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2532323 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2532333 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2532415 WARN (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2532417 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2532435 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2532435 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 2532436 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7ebb4ac1
[junit4] 2> 2532438 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=11, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=78.751953125, floorSegmentMB=1.8701171875, forceMergeDeletesPctAllowed=26.71723740692539, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6695027851037475
[junit4] 2> 2532503 WARN (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2532518 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2532518 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2532518 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2532518 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2532519 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1097877596, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2532520 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@460f00a5[collection1] main]
[junit4] 2> 2532521 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2532522 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2532522 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2532523 INFO (searcherExecutor-8219-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 c:collection1) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@460f00a5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2532524 INFO (coreLoadExecutor-8218-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555914567531364352
[junit4] 2> 2532528 INFO (coreZkRegister-8213-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 c:collection1) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2532529 INFO (updateExecutor-3399-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2532529 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2532529 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2532529 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2532529 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2532529 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:54258/sovt/cq/collection1/] and I am [http://127.0.0.1:54262/sovt/cq/collection1/]
[junit4] 2> 2532532 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:54258/sovt/cq]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:54262_sovt%252Fcq&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2532535 INFO (qtp437457824-18013) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2532536 INFO (qtp437457824-18013) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 2532536 INFO (qtp437457824-18013) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:54262_sovt%2Fcq, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:54262/sovt/cq","node_name":"127.0.0.1:54262_sovt%2Fcq","state":"down"}
[junit4] 2> 2533034 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-3-001/cores/collection1
[junit4] 2> 2533036 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-3-001
[junit4] 2> 2533037 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2533039 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@467a26c3{/sovt/cq,null,AVAILABLE}
[junit4] 2> 2533040 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@90da241{HTTP/1.1,[http/1.1]}{127.0.0.1:54267}
[junit4] 2> 2533041 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.e.j.s.Server Started @2539631ms
[junit4] 2> 2533041 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/sovt/cq, hostPort=54267, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-3-001/cores}
[junit4] 2> 2533041 ERROR (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2533041 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2533041 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2533042 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2533042 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-08T00:36:02.095Z
[junit4] 2> 2533046 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2533046 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-3-001/solr.xml
[junit4] 2> 2533122 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54240/solr
[junit4] 2> 2533142 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54267_sovt%2Fcq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2533152 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54267_sovt%2Fcq ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54267_sovt%2Fcq
[junit4] 2> 2533155 INFO (zkCallback-3396-thread-1-processing-n:127.0.0.1:54258_sovt%2Fcq) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2533155 INFO (zkCallback-3387-thread-1-processing-n:127.0.0.1:54248_sovt%2Fcq) [n:127.0.0.1:54248_sovt%2Fcq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2533155 INFO (zkCallback-3402-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq) [n:127.0.0.1:54262_sovt%2Fcq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2533156 INFO (zkCallback-3391-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2533157 INFO (zkCallback-3409-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2533222 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54267_sovt%2Fcq ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-3-001/cores
[junit4] 2> 2533222 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [n:127.0.0.1:54267_sovt%2Fcq ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2533225 INFO (OverseerStateUpdate-97244659726548996-127.0.0.1:54248_sovt%2Fcq-n_0000000000) [n:127.0.0.1:54248_sovt%2Fcq ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2533536 INFO (qtp437457824-18013) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:54262_sovt%2Fcq, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:54262/sovt/cq","node_name":"127.0.0.1:54262_sovt%2Fcq","state":"recovering"}
[junit4] 2> 2533537 INFO (qtp437457824-18013) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2533537 INFO (qtp437457824-18013) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:54262_sovt%252Fcq&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1001
[junit4] 2> 2534255 WARN (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2534256 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2534268 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2534374 WARN (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2534376 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2534400 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2534401 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 2534401 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7ebb4ac1
[junit4] 2> 2534404 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=11, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=78.751953125, floorSegmentMB=1.8701171875, forceMergeDeletesPctAllowed=26.71723740692539, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6695027851037475
[junit4] 2> 2534478 WARN (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2534495 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2534496 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2534496 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2534496 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2534498 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1097877596, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2534498 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@59bd8ee7[collection1] main]
[junit4] 2> 2534500 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2534501 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2534501 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2534502 INFO (searcherExecutor-8230-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 c:collection1) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@59bd8ee7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2534502 INFO (coreLoadExecutor-8229-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555914569605447680
[junit4] 2> 2534508 INFO (coreZkRegister-8224-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 c:collection1) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2534508 INFO (updateExecutor-3406-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2534509 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2534509 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2534510 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2534510 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2534510 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:54258/sovt/cq/collection1/] and I am [http://127.0.0.1:54267/sovt/cq/collection1/]
[junit4] 2> 2534512 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:54258/sovt/cq]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:54267_sovt%252Fcq&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2534513 INFO (qtp437457824-18017) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2534514 INFO (qtp437457824-18017) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 2534514 INFO (qtp437457824-18017) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:54267_sovt%2Fcq, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:54267/sovt/cq","node_name":"127.0.0.1:54267_sovt%2Fcq","state":"down"}
[junit4] 2> 2534585 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2534586 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30 for each attempt
[junit4] 2> 2534586 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 2535515 INFO (qtp437457824-18017) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:54267_sovt%2Fcq, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:54267/sovt/cq","node_name":"127.0.0.1:54267_sovt%2Fcq","state":"recovering"}
[junit4] 2> 2535515 INFO (qtp437457824-18017) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2535515 INFO (qtp437457824-18017) [n:127.0.0.1:54258_sovt%2Fcq ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:54267_sovt%252Fcq&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1001
[junit4] 2> 2540539 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:54258/sovt/cq/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2540539 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:54262/sovt/cq START replicas=[http://127.0.0.1:54258/sovt/cq/collection1/] nUpdates=1000
[junit4] 2> 2540541 INFO (qtp437457824-18014) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2540541 INFO (qtp437457824-18014) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp=/sovt/cq path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2540542 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2540542 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2540542 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2540542 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2540543 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2540543 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2540543 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2540543 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2540543 INFO (recoveryExecutor-3400-thread-1-processing-n:127.0.0.1:54262_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2542526 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:54258/sovt/cq/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2542526 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:54267/sovt/cq START replicas=[http://127.0.0.1:54258/sovt/cq/collection1/] nUpdates=1000
[junit4] 2> 2542528 INFO (qtp437457824-18015) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2542528 INFO (qtp437457824-18015) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp=/sovt/cq path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2542529 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2542529 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2542529 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2542529 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2542530 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2542530 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2542530 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2542530 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2542530 INFO (recoveryExecutor-3407-thread-1-processing-n:127.0.0.1:54267_sovt%2Fcq x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2542616 INFO (TEST-PeerSyncReplicationTest.test-seed#[46C8FCAE122A307B]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 2542620 INFO (qtp855186579-17976) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2542620 INFO (qtp855186579-17976) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2542621 INFO (qtp855186579-17976) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2542621 INFO (qtp855186579-17976) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 1
[junit4] 2> 2542624 INFO (qtp437457824-18013) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2542624 INFO (qtp437457824-18013) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2542626 INFO (qtp437457824-18013) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2542626 INFO (qtp437457824-18013) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:54258/sovt/cq/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 2542626 INFO (qtp1450414870-18076) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2542626 INFO (qtp1450414870-18076) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2542626 INFO (qtp465254745-18041) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2542626 INFO (qtp465254745-18041) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2542627 INFO (qtp1450414870-18076) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2542627 INFO (qtp1450414870-18076) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:54258/sovt/cq/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
[junit4] 2> 2542627 INFO (qtp465254745-18041) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2542627 INFO (qtp465254745-18041) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:54258/sovt/cq/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 2542628 INFO (qtp437457824-18018) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 5
[junit4] 2> 2542630 INFO (qtp437457824-18014) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp=/sovt/cq path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2542632 INFO (qtp465254745-18043) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.S.Request [collection1] webapp=/sovt/cq path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2542633 INFO (qtp1450414870-18078) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.S.Request [collection1] webapp=/sovt/cq path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2544647 INFO (qtp855186579-17977) [n:127.0.0.1:54248_sovt%2Fcq c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1555914580241154048)} 0 2
[junit4] 2> 2544651 INFO (qtp1450414870-18079) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={update.distrib=FROMLEADER&_version_=-1555914580244299776&distrib.from=http://127.0.0.1:54258/sovt/cq/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1555914580244299776)} 0 1
[junit4] 2> 2544652 INFO (qtp465254745-18044) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={update.distrib=FROMLEADER&_version_=-1555914580244299776&distrib.from=http://127.0.0.1:54258/sovt/cq/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1555914580244299776)} 0 1
[junit4] 2> 2544652 INFO (qtp437457824-18016) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1555914580244299776)} 0 3
[junit4] 2> 2544660 INFO (qtp1450414870-18080) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54258/sovt/cq/collection1/&wt=javabin&version=2}{add=[0 (1555914580251639808)]} 0 1
[junit4] 2> 2544660 INFO (qtp465254745-18045) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54258/sovt/cq/collection1/&wt=javabin&version=2}{add=[0 (1555914580251639808)]} 0 1
[junit4] 2> 2544660 INFO (qtp437457824-18015) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={wt=javabin&version=2}{add=[0 (1555914580251639808)]} 0 4
[junit4] 2> 2544662 INFO (qtp1450414870-18081) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54258/sovt/cq/collection1/&wt=javabin&version=2}{add=[1 (1555914580257931264)]} 0 0
[junit4] 2> 2544662 INFO (qtp465254745-18046) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54258/sovt/cq/collection1/&wt=javabin&version=2}{add=[1 (1555914580257931264)]} 0 0
[junit4] 2> 2544662 INFO (qtp437457824-18017) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={wt=javabin&version=2}{add=[1 (1555914580257931264)]} 0 1
[junit4] 2> 2544664 INFO (qtp1450414870-18074) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54258/sovt/cq/collection1/&wt=javabin&version=2}{add=[2 (1555914580260028416)]} 0 0
[junit4] 2> 2544664 INFO (qtp465254745-18039) [n:127.0.0.1:54262_sovt%2Fcq c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54258/sovt/cq/collection1/&wt=javabin&version=2}{add=[2 (1555914580260028416)]} 0 0
[junit4] 2> 2544664 INFO (qtp437457824-18013) [n:127.0.0.1:54258_sovt%2Fcq c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={wt=javabin&version=2}{add=[2 (1555914580260028416)]} 0 1
[junit4] 2> 2544666 INFO (qtp1450414870-18076) [n:127.0.0.1:54267_sovt%2Fcq c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/sovt/cq path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:54258/sovt/cq/collection1/&wt=javabin&version=2}{add=[3 (1555914580262125568)]} 0 0
[junit4]
[...truncated too long message...]
6 /solr/aliases.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 2 /solr/overseer_elect/election/97244659726549000-127.0.0.1:54258_sovt%2Fcq-n_0000000001
[junit4] 2> 2 /solr/collections/collection1/leader_elect/shard1/election/97244659726549000-core_node1-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 207 /solr/overseer/collection-queue-work
[junit4] 2> 43 /solr/overseer/queue
[junit4] 2> 6 /solr/collections
[junit4] 2> 5 /solr/live_nodes
[junit4] 2> 4 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=PeerSyncReplicationTest -Dtests.method=test -Dtests.seed=46C8FCAE122A307B -Dtests.slow=true -Dtests.locale=et-EE -Dtests.timezone=Europe/Brussels -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] FAILURE 215s J1 | PeerSyncReplicationTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: timeout waiting to see all nodes active
[junit4] > at __randomizedtesting.SeedInfo.seed([46C8FCAE122A307B:CE9CC374BCD65D83]:0)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:311)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:262)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:244)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:133)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2736594 INFO (SUITE-PeerSyncReplicationTest-seed#[46C8FCAE122A307B]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_46C8FCAE122A307B-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {other_tl1=BlockTreeOrds(blocksize=128), range_facet_l_dv=PostingsFormat(name=LuceneVarGapDocFreqInterval), rnd_s=BlockTreeOrds(blocksize=128), multiDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), intDefault=BlockTreeOrds(blocksize=128), a_i1=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), range_facet_l=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), _version_=BlockTreeOrds(blocksize=128), a_t=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), id=PostingsFormat(name=LuceneVarGapDocFreqInterval), range_facet_i_dv=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), text=FST50, timestamp=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{range_facet_l_dv=DocValuesFormat(name=Direct), range_facet_i_dv=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=1402, maxMBSortInHeap=5.320134873018556, sim=RandomSimilarity(queryNorm=true): {}, locale=et-EE, timezone=Europe/Brussels
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=3,threads=1,free=162670032,total=431411200
[junit4] 2> NOTE: All tests run in this JVM: [TestMinMaxOnMultiValuedField, TestSolrFieldCacheMBean, TestQueryUtils, TestMacroExpander, TestSearcherReuse, BasicAuthStandaloneTest, TestNumericTerms64, SortByFunctionTest, TestFieldCacheVsDocValues, TestFieldTypeResource, NoCacheHeaderTest, TestRandomFlRTGCloud, SolrMetricReporterTest, TestLegacyFieldCache, InfoHandlerTest, TestAuthenticationFramework, BlockJoinFacetDistribTest, TestEmbeddedSolrServerConstructors, TestBinaryResponseWriter, CollectionsAPISolrJTest, TestQuerySenderListener, AddSchemaFieldsUpdateProcessorFactoryTest, EnumFieldTest, TestUtils, CollectionTooManyReplicasTest, WordBreakSolrSpellCheckerTest, TestScoreJoinQPNoScore, MinimalSchemaTest, RegexBoostProcessorTest, TestDocBasedVersionConstraints, TestInitQParser, TestManagedSchemaAPI, CdcrReplicationHandlerTest, TestSortByMinMaxFunction, CollectionsAPIDistributedZkTest, TestPHPSerializedResponseWriter, DeleteShardTest, TestStressUserVersions, WrapperMergePolicyFactoryTest, ShufflingReplicaListTransformerTest, TestQueryWrapperFilter, RulesTest, TestRealTimeGet, TestRecoveryHdfs, RequestHandlersTest, TestTestInjection, JSONWriterTest, BigEndianAscendingWordDeserializerTest, CacheHeaderTest, TestRecovery, TestDFISimilarityFactory, TestCollapseQParserPlugin, HdfsChaosMonkeyNothingIsSafeTest, DateMathParserTest, TestSolrDynamicMBean, TestHdfsUpdateLog, TestRestManager, BadIndexSchemaTest, TestDelegationWithHadoopAuth, JsonLoaderTest, TestSolrQueryParserDefaultOperatorResource, TestCorePropertiesReload, CoreAdminRequestStatusTest, FieldAnalysisRequestHandlerTest, DeleteNodeTest, TestFuzzyAnalyzedSuggestions, TestCollationFieldDocValues, PeerSyncWithIndexFingerprintCachingTest, NotRequiredUniqueKeyTest, CdcrUpdateLogTest, DistributedQueryComponentOptimizationTest, TermVectorComponentTest, FullHLLTest, StatsComponentTest, OverseerModifyCollectionTest, HdfsRestartWhileUpdatingTest, CdcrVersionReplicationTest, MultiThreadedOCPTest, TestJsonFacetRefinement, TestSolrQueryParser, TestFiltering, TestClusterProperties, TestXmlQParser, SolrCoreMetricManagerTest, DeleteInactiveReplicaTest, CursorMarkTest, TestDistributedMissingSort, TestUniqueKeyFieldResource, TestExportWriter, HdfsThreadLeakTest, ExitableDirectoryReaderTest, TestComponentsName, ResponseHeaderTest, TestLMDirichletSimilarityFactory, TestFastWriter, TestSolrCoreProperties, DebugComponentTest, TestAnalyzedSuggestions, RAMDirectoryFactoryTest, TestChildDocTransformer, TestNRTOpen, SimpleMLTQParserTest, ConvertedLegacyTest, TestRandomFaceting, CdcrBootstrapTest, FacetPivotSmallTest, TestDefaultSearchFieldResource, OverseerStatusTest, TestIndexingPerformance, TestDeleteCollectionOnDownNodes, HighlighterConfigTest, RemoteQueryErrorTest, LeaderElectionContextKeyTest, DistanceFunctionTest, TestDefaultStatsCache, TestSchemaManager, TestComplexPhraseQParserPlugin, DistributedExpandComponentTest, CoreSorterTest, TestStressRecovery, SuggesterWFSTTest, CreateCollectionCleanupTest, SolrCoreCheckLockOnStartupTest, FileBasedSpellCheckerTest, TestSmileRequest, FieldMutatingUpdateProcessorTest, TestSolrCloudSnapshots, DistributedVersionInfoTest, AnalyticsMergeStrategyTest, RequestLoggingTest, QueryParsingTest, TestRTimerTree, SolrJmxReporterTest, DistribCursorPagingTest, TestStressLiveNodes, DefaultValueUpdateProcessorTest, ResponseLogComponentTest, CollectionReloadTest, TestDynamicFieldResource, SpellCheckCollatorTest, HdfsDirectoryTest, LeaderElectionIntegrationTest, TestConfigSetsAPIZkFailure, TestPerFieldSimilarityWithDefaultOverride, TestIndexSearcher, TestUnifiedSolrHighlighter, SpellPossibilityIteratorTest, TestFieldResource, TestLuceneMatchVersion, TestSchemaVersionResource, ClassificationUpdateProcessorFactoryTest, TestJsonRequest, TestFileDictionaryLookup, SpellCheckComponentTest, TestDynamicLoading, TestConfigOverlay, AddBlockUpdateTest, DistributedFacetPivotSmallAdvancedTest, TestWriterPerf, AutoCommitTest, JavabinLoaderTest, DistributedIntervalFacetingTest, SaslZkACLProviderTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, OverseerTest, BasicZkTest, FullSolrCloudDistribCmdsTest, ClusterStateUpdateTest, ZkSolrClientTest, TestZkChroot, TestRandomDVFaceting, ShardRoutingCustomTest, TestDistributedGrouping, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, ZkControllerTest, TestJoin, TestReload, HardAutoCommitTest, DistributedTermsComponentTest, TestRangeQuery, SimpleFacetsTest, TestSolr4Spatial, SolrCoreTest, TestGroupingSearch, SolrCmdDistributorTest, QueryElevationComponentTest, BasicFunctionalityTest, ShowFileRequestHandlerTest, SimplePostToolTest, TestCoreDiscovery, TestExtendedDismaxParser, CoreAdminHandlerTest, SuggesterTSTTest, SuggesterTest, TestStressLucene, TestTrie, SpatialFilterTest, PolyFieldTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, TestRemoteStreaming, SolrInfoMBeanTest, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, PrimitiveFieldTypeTest, TestValueSourceCache, MoreLikeThisHandlerTest, LoggingHandlerTest, TestJmxIntegration, UpdateRequestProcessorFactoryTest, TestPartialUpdateDeduplication, TestLFUCache, TestDocSet, TestConfig, DirectSolrConnectionTest, TestCharFilters, TestXIncludeConfig, EchoParamsTest, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestLMJelinekMercerSimilarityFactory, TestJmxMonitoredMap, URLClassifyProcessorTest, TestFastLRUCache, TestSystemIdResolver, PrimUtilsTest, SpellingQueryConverterTest, ClusterStateTest, TestLRUCache, SliceStateTest, UUIDFieldTest, CircularListTest, CursorPagingTest, CloudExitableDirectoryReaderTest, CollectionStateFormat2Test, CustomCollectionTest, DistribJoinFromCollectionTest, ForceLeaderTest, PeerSyncReplicationTest]
[junit4] Completed [510/676 (1!)] on J1 in 215.46s, 1 test, 1 failure <<< FAILURES!
[...truncated 62991 lines...]