You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/01/25 16:19:45 UTC
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3797
- Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3797/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
timeout waiting to see all nodes active
Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
at __randomizedtesting.SeedInfo.seed([A06A4D09107E4409:283E72D3BE8229F1]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 11457 lines...]
[junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/init-core-data-001
[junit4] 2> 1019284 INFO (SUITE-PeerSyncReplicationTest-seed#[A06A4D09107E4409]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields
[junit4] 2> 1019288 INFO (SUITE-PeerSyncReplicationTest-seed#[A06A4D09107E4409]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 1019289 INFO (SUITE-PeerSyncReplicationTest-seed#[A06A4D09107E4409]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 1019290 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1019291 INFO (Thread-1655) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1019291 INFO (Thread-1655) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1019395 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ZkTestServer start zk server on port:60300
[junit4] 2> 1019425 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1019432 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 1019436 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1019440 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1019443 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1019446 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1019449 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1019453 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1019456 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1019459 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1019463 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1023112 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/control-001/cores/collection1
[junit4] 2> 1023115 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1023117 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4d355d8e{/,null,AVAILABLE}
[junit4] 2> 1023119 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5d1fcf0a{HTTP/1.1,[http/1.1]}{127.0.0.1:60303}
[junit4] 2> 1023119 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server Started @1028156ms
[junit4] 2> 1023119 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/tempDir-001/control/data, hostContext=/, hostPort=60303, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/control-001/cores}
[junit4] 2> 1023119 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1023119 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1023120 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1023120 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1023120 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T15:11:30.446Z
[junit4] 2> 1023124 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1023124 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/control-001/solr.xml
[junit4] 2> 1023138 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60300/solr
[junit4] 2> 1023184 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60303_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60303_
[junit4] 2> 1023186 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60303_ ] o.a.s.c.Overseer Overseer (id=97344362028793860-127.0.0.1:60303_-n_0000000000) starting
[junit4] 2> 1023211 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60303_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60303_
[junit4] 2> 1023214 INFO (OverseerStateUpdate-97344362028793860-127.0.0.1:60303_-n_0000000000) [n:127.0.0.1:60303_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1023358 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60303_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/control-001/cores
[junit4] 2> 1023358 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60303_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1023363 INFO (OverseerStateUpdate-97344362028793860-127.0.0.1:60303_-n_0000000000) [n:127.0.0.1:60303_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1024386 WARN (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 1024387 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1024399 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1024486 WARN (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1024488 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1024519 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 1024519 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/control-001/cores/collection1/data/]
[junit4] 2> 1024519 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d0b4029
[junit4] 2> 1024525 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35301399807770073]
[junit4] 2> 1024536 WARN (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1024552 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1024552 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1024554 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1024554 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1024554 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=76.486328125, floorSegmentMB=0.50390625, forceMergeDeletesPctAllowed=17.875198882979568, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7157930179055988
[junit4] 2> 1024555 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@c3aa56[collection1] main]
[junit4] 2> 1024557 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1024558 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1024558 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1024559 INFO (searcherExecutor-3457-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@c3aa56[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1024559 INFO (coreLoadExecutor-3456-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557509797980405760
[junit4] 2> 1024570 INFO (coreZkRegister-3449-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1024570 INFO (coreZkRegister-3449-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1024570 INFO (coreZkRegister-3449-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60303/collection1/
[junit4] 2> 1024570 INFO (coreZkRegister-3449-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1024570 INFO (coreZkRegister-3449-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:60303/collection1/ has no replicas
[junit4] 2> 1024577 INFO (coreZkRegister-3449-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60303/collection1/ shard1
[junit4] 2> 1024712 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1024715 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60300/solr ready
[junit4] 2> 1024715 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1024715 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 1024737 INFO (coreZkRegister-3449-thread-1-processing-n:127.0.0.1:60303_ x:collection1 c:control_collection) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1025417 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-1-001/cores/collection1
[junit4] 2> 1025419 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-1-001
[junit4] 2> 1025420 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1025421 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@581b9b23{/,null,AVAILABLE}
[junit4] 2> 1025422 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1787576a{HTTP/1.1,[http/1.1]}{127.0.0.1:60309}
[junit4] 2> 1025422 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server Started @1030459ms
[junit4] 2> 1025422 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=60309, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-1-001/cores}
[junit4] 2> 1025422 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1025423 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1025423 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1025423 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1025423 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T15:11:32.749Z
[junit4] 2> 1025428 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1025428 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-1-001/solr.xml
[junit4] 2> 1025454 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60300/solr
[junit4] 2> 1025474 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1025483 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60309_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60309_
[junit4] 2> 1025488 INFO (zkCallback-1484-thread-2-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1025488 INFO (zkCallback-1488-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1025490 INFO (zkCallback-1494-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1025577 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60309_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-1-001/cores
[junit4] 2> 1025577 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60309_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1025583 INFO (OverseerStateUpdate-97344362028793860-127.0.0.1:60303_-n_0000000000) [n:127.0.0.1:60303_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1025696 INFO (zkCallback-1494-thread-2-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1026719 WARN (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 1026720 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1026733 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1026824 WARN (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1026826 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1026846 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1026846 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 1026846 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d0b4029
[junit4] 2> 1026851 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35301399807770073]
[junit4] 2> 1026967 WARN (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1026988 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1026988 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1026989 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1026989 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1026989 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=76.486328125, floorSegmentMB=0.50390625, forceMergeDeletesPctAllowed=17.875198882979568, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7157930179055988
[junit4] 2> 1026990 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@67247e01[collection1] main]
[junit4] 2> 1026992 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1026993 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1026993 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1026994 INFO (searcherExecutor-3468-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@67247e01[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1026995 INFO (coreLoadExecutor-3467-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557509800534736896
[junit4] 2> 1027005 INFO (coreZkRegister-3462-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1027005 INFO (coreZkRegister-3462-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1027005 INFO (coreZkRegister-3462-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60309/collection1/
[junit4] 2> 1027005 INFO (coreZkRegister-3462-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1027005 INFO (coreZkRegister-3462-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:60309/collection1/ has no replicas
[junit4] 2> 1027013 INFO (coreZkRegister-3462-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60309/collection1/ shard1
[junit4] 2> 1027120 INFO (zkCallback-1494-thread-2-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1027166 INFO (coreZkRegister-3462-thread-1-processing-n:127.0.0.1:60309_ x:collection1 c:collection1) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1027280 INFO (zkCallback-1494-thread-2-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1027774 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-2-001/cores/collection1
[junit4] 2> 1027776 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-2-001
[junit4] 2> 1027776 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1027780 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@54caacac{/,null,AVAILABLE}
[junit4] 2> 1027781 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@13423b1e{HTTP/1.1,[http/1.1]}{127.0.0.1:60313}
[junit4] 2> 1027781 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server Started @1032819ms
[junit4] 2> 1027781 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=60313, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-2-001/cores}
[junit4] 2> 1027781 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1027784 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1027784 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1027785 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1027785 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T15:11:35.111Z
[junit4] 2> 1027789 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1027790 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-2-001/solr.xml
[junit4] 2> 1027806 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60300/solr
[junit4] 2> 1027832 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1027853 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60313_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60313_
[junit4] 2> 1027857 INFO (zkCallback-1494-thread-2-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1027857 INFO (zkCallback-1488-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1027857 INFO (zkCallback-1484-thread-2-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1027859 INFO (zkCallback-1500-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1027940 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60313_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-2-001/cores
[junit4] 2> 1027940 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60313_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1027969 INFO (OverseerStateUpdate-97344362028793860-127.0.0.1:60303_-n_0000000000) [n:127.0.0.1:60303_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1028096 INFO (zkCallback-1500-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1028096 INFO (zkCallback-1494-thread-2-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1028985 WARN (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 1028986 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1029001 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1029115 WARN (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1029117 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1029142 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1029143 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 1029143 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d0b4029
[junit4] 2> 1029148 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35301399807770073]
[junit4] 2> 1029159 WARN (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1029179 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1029179 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1029180 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1029180 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1029182 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=76.486328125, floorSegmentMB=0.50390625, forceMergeDeletesPctAllowed=17.875198882979568, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7157930179055988
[junit4] 2> 1029183 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@57163345[collection1] main]
[junit4] 2> 1029187 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1029188 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1029188 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1029190 INFO (searcherExecutor-3479-thread-1-processing-n:127.0.0.1:60313_ x:collection1 c:collection1) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@57163345[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1029191 INFO (coreLoadExecutor-3478-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557509802837409792
[junit4] 2> 1029198 INFO (coreZkRegister-3473-thread-1-processing-n:127.0.0.1:60313_ x:collection1 c:collection1) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 1029198 INFO (updateExecutor-1497-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 1029199 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 1029199 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 1029199 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 1029199 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 1029199 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:60309/collection1/] and I am [http://127.0.0.1:60313/collection1/]
[junit4] 2> 1029202 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:60309]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:60313_&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 1029206 INFO (qtp1284525073-8013) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 1029207 INFO (qtp1284525073-8013) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 1029207 INFO (qtp1284525073-8013) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:60313_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:60313","node_name":"127.0.0.1:60313_","state":"down"}
[junit4] 2> 1029310 INFO (zkCallback-1500-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1029310 INFO (zkCallback-1494-thread-2-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1029616 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-3-001/cores/collection1
[junit4] 2> 1029618 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-3-001
[junit4] 2> 1029618 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1029620 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3dba342c{/,null,AVAILABLE}
[junit4] 2> 1029620 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@b1921c{HTTP/1.1,[http/1.1]}{127.0.0.1:60318}
[junit4] 2> 1029620 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.Server Started @1034658ms
[junit4] 2> 1029620 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=60318, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-3-001/cores}
[junit4] 2> 1029621 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1029622 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 1029622 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1029622 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1029622 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T15:11:36.948Z
[junit4] 2> 1029626 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1029626 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-3-001/solr.xml
[junit4] 2> 1029648 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60300/solr
[junit4] 2> 1029669 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60318_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1029678 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60318_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60318_
[junit4] 2> 1029682 INFO (zkCallback-1500-thread-1-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1029683 INFO (zkCallback-1484-thread-1-processing-n:127.0.0.1:60303_) [n:127.0.0.1:60303_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1029683 INFO (zkCallback-1494-thread-2-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1029683 INFO (zkCallback-1488-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1029684 INFO (zkCallback-1507-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1029733 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60318_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-3-001/cores
[junit4] 2> 1029733 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [n:127.0.0.1:60318_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1029737 INFO (OverseerStateUpdate-97344362028793860-127.0.0.1:60303_-n_0000000000) [n:127.0.0.1:60303_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1029850 INFO (zkCallback-1494-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1029850 INFO (zkCallback-1507-thread-2-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1029850 INFO (zkCallback-1500-thread-2-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1030210 INFO (qtp1284525073-8013) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:60313_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:60313","node_name":"127.0.0.1:60313_","state":"recovering"}
[junit4] 2> 1030210 INFO (qtp1284525073-8013) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 1030210 INFO (qtp1284525073-8013) [n:127.0.0.1:60309_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:60313_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1003
[junit4] 2> 1030759 WARN (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 1030759 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1030781 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1030895 WARN (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 1030897 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1030921 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 1030921 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 1030922 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d0b4029
[junit4] 2> 1030928 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.35301399807770073]
[junit4] 2> 1030937 WARN (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1030953 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1030953 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1030954 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1030954 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1030955 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=76.486328125, floorSegmentMB=0.50390625, forceMergeDeletesPctAllowed=17.875198882979568, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7157930179055988
[junit4] 2> 1030956 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@17cc6ccc[collection1] main]
[junit4] 2> 1030958 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1030959 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1030959 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1030960 INFO (searcherExecutor-3490-thread-1-processing-n:127.0.0.1:60318_ x:collection1 c:collection1) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@17cc6ccc[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1030961 INFO (coreLoadExecutor-3489-thread-1-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557509804693389312
[junit4] 2> 1030967 INFO (coreZkRegister-3484-thread-1-processing-n:127.0.0.1:60318_ x:collection1 c:collection1) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 1030967 INFO (updateExecutor-1504-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 1030968 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 1030968 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 1030968 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 1030968 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 1030968 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:60309/collection1/] and I am [http://127.0.0.1:60318/collection1/]
[junit4] 2> 1030973 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:60309]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:60318_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 1030975 INFO (qtp1284525073-8017) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 1030976 INFO (qtp1284525073-8017) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 1030976 INFO (qtp1284525073-8017) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:60318_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:60318","node_name":"127.0.0.1:60318_","state":"down"}
[junit4] 2> 1031079 INFO (zkCallback-1507-thread-2-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1031079 INFO (zkCallback-1500-thread-2-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1031079 INFO (zkCallback-1494-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1031091 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 1031091 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30 for each attempt
[junit4] 2> 1031091 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 1031983 INFO (qtp1284525073-8017) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:60318_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:60318","node_name":"127.0.0.1:60318_","state":"recovering"}
[junit4] 2> 1031984 INFO (qtp1284525073-8017) [n:127.0.0.1:60309_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 1031984 INFO (qtp1284525073-8017) [n:127.0.0.1:60309_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:60318_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1008
[junit4] 2> 1037211 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:60309/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 1037211 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:60313 START replicas=[http://127.0.0.1:60309/collection1/] nUpdates=1000
[junit4] 2> 1037214 INFO (qtp1284525073-8014) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1037214 INFO (qtp1284525073-8014) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 1037215 INFO (recoveryExecutor-1498-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 1037220 INFO (zkCallback-1500-thread-2-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1037220 INFO (zkCallback-1507-thread-2-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1037220 INFO (zkCallback-1494-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1038988 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:60309/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 1038988 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:60318 START replicas=[http://127.0.0.1:60309/collection1/] nUpdates=1000
[junit4] 2> 1038990 INFO (qtp1284525073-8015) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1038990 INFO (qtp1284525073-8015) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1038991 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1038991 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 1038991 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1038991 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1038991 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1038991 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 1038991 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 1038992 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 1038992 INFO (recoveryExecutor-1505-thread-1-processing-n:127.0.0.1:60318_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 1038996 INFO (zkCallback-1500-thread-2-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1038996 INFO (zkCallback-1494-thread-1-processing-n:127.0.0.1:60309_) [n:127.0.0.1:60309_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1038996 INFO (zkCallback-1507-thread-2-processing-n:127.0.0.1:60318_) [n:127.0.0.1:60318_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1039128 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 1039134 INFO (qtp1186743350-7973) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1039134 INFO (qtp1186743350-7973) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1039135 INFO (qtp1186743350-7973) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1039135 INFO (qtp1186743350-7973) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 1
[junit4] 2> 1039139 INFO (qtp1284525073-8018) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1039139 INFO (qtp1284525073-8018) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1039140 INFO (qtp1284525073-8018) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1039141 INFO (qtp1284525073-8018) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:60309/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 1039142 INFO (qtp24934752-8043) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1039142 INFO (qtp294505202-8078) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1039142 INFO (qtp24934752-8043) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1039142 INFO (qtp294505202-8078) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1039143 INFO (qtp24934752-8043) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1039143 INFO (qtp24934752-8043) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:60309/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
[junit4] 2> 1039143 INFO (qtp294505202-8078) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1039143 INFO (qtp294505202-8078) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:60309/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
[junit4] 2> 1039143 INFO (qtp1284525073-8017) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={_stateVer_=collection1:9&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 6
[junit4] 2> 1039146 INFO (qtp1284525073-8011) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 1039147 INFO (qtp24934752-8045) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 1039148 INFO (qtp294505202-8080) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 1041162 INFO (qtp1186743350-7974) [n:127.0.0.1:60303_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557509815387815936)} 0 2
[junit4] 2> 1041167 INFO (qtp294505202-8081) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1557509815390961664&distrib.from=http://127.0.0.1:60309/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557509815390961664)} 0 1
[junit4] 2> 1041167 INFO (qtp24934752-8046) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1557509815390961664&distrib.from=http://127.0.0.1:60309/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557509815390961664)} 0 1
[junit4] 2> 1041168 INFO (qtp1284525073-8014) [n:127.0.0.1:60309_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557509815390961664)} 0 4
[junit4] 2> 1041184 INFO (qtp294505202-8082) [n:127.0.0.1:60318_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:60309/collection1/&wt=javabin&version=2}{add=[0 (1557509815402496000)]} 0 4
[...truncated too long message...]
re_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Skipping recovery because Solr is shutdown
[junit4] 2> 1228046 INFO (zkCallback-1514-thread-4-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1232218 INFO (recoveryExecutor-1512-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy RecoveryStrategy has been closed
[junit4] 2> 1232218 INFO (recoveryExecutor-1512-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[false]
[junit4] 2> 1232218 INFO (recoveryExecutor-1512-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@3f98865
[junit4] 2> 1232219 WARN (recoveryExecutor-1512-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1] coreNodeName=[core_node2]
[junit4] 2> 1232219 INFO (zkCallback-1514-thread-3-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1232250 INFO (recoveryExecutor-1512-thread-1-processing-n:127.0.0.1:60313_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:60313_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 1232250 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.Overseer Overseer (id=97344362028793874-127.0.0.1:60313_-n_0000000004) closing
[junit4] 2> 1232251 INFO (OverseerStateUpdate-97344362028793874-127.0.0.1:60313_-n_0000000004) [n:127.0.0.1:60313_ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:60313_
[junit4] 2> 1232252 WARN (zkCallback-1514-thread-3-processing-n:127.0.0.1:60313_) [n:127.0.0.1:60313_ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1232253 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 1232254 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@1bbcfd62{HTTP/1.1,[http/1.1]}{127.0.0.1:60313}
[junit4] 2> 1232254 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1f673412{/,null,UNAVAILABLE}
[junit4] 2> 1232255 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 60318
[junit4] 2> 1232257 INFO (TEST-PeerSyncReplicationTest.test-seed#[A06A4D09107E4409]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60300 60300
[junit4] 2> 1232274 INFO (Thread-1655) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60300 60300
[junit4] 2> 1236047 WARN (Thread-1655) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2> 2 /solr/collections/collection1/leader_elect/shard1/election/97344362028793865-core_node1-n_0000000000
[junit4] 2> 2 /solr/overseer_elect/election/97344362028793865-127.0.0.1:60309_-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 207 /solr/overseer/collection-queue-work
[junit4] 2> 48 /solr/overseer/queue
[junit4] 2> 16 /solr/overseer/queue-work
[junit4] 2> 6 /solr/collections
[junit4] 2> 5 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=PeerSyncReplicationTest -Dtests.method=test -Dtests.seed=A06A4D09107E4409 -Dtests.slow=true -Dtests.locale=ar-DZ -Dtests.timezone=PRC -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] FAILURE 217s J0 | PeerSyncReplicationTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: timeout waiting to see all nodes active
[junit4] > at __randomizedtesting.SeedInfo.seed([A06A4D09107E4409:283E72D3BE8229F1]:0)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 1236113 INFO (SUITE-PeerSyncReplicationTest-seed#[A06A4D09107E4409]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.PeerSyncReplicationTest_A06A4D09107E4409-001
[junit4] 2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=7, maxDocsPerChunk=145, blockSize=6), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=7, blockSize=6)), sim=RandomSimilarity(queryNorm=true): {}, locale=ar-DZ, timezone=PRC
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=58012672,total=385875968
[junit4] 2> NOTE: All tests run in this JVM: [TestRuleBasedAuthorizationPlugin, TestStressReorder, TestPointFields, SolrRequestParserTest, PeerSyncTest, TestDocBasedVersionConstraints, TestStressUserVersions, OverseerStatusTest, DistributedQueryComponentOptimizationTest, TestAnalyzeInfixSuggestions, TestPerFieldSimilarity, ReplicationFactorTest, DocValuesTest, SolrCoreTest, DirectUpdateHandlerTest, SparseHLLTest, ClassificationUpdateProcessorFactoryTest, TestLegacyFieldCache, ZkNodePropsTest, TestHdfsUpdateLog, TestJsonFacetRefinement, TestOverriddenPrefixQueryForCustomFieldType, TestDFISimilarityFactory, BlockJoinFacetDistribTest, PreAnalyzedFieldManagedSchemaCloudTest, TestExactSharedStatsCache, TestSQLHandlerNonCloud, SolrCloudExampleTest, SolrGangliaReporterTest, DistributedQueryElevationComponentTest, TestSolrCloudSnapshots, TestSSLRandomization, TestFieldCacheVsDocValues, TestWordDelimiterFilterFactory, TestDynamicLoading, NumericFieldsTest, TestSurroundQueryParser, HdfsCollectionsAPIDistributedZkTest, SecurityConfHandlerTest, CdcrRequestHandlerTest, TestFastLRUCache, TestPushWriter, TestFilteredDocIdSet, TestHashQParserPlugin, DateFieldTest, MultiTermTest, TestDistributedSearch, TestSolrCloudWithSecureImpersonation, RollingRestartTest, HdfsChaosMonkeyNothingIsSafeTest, TestFastOutputStream, HdfsRecoverLeaseTest, JSONWriterTest, ZkCLITest, DOMUtilTest, TestDownShardTolerantSearch, ZkStateReaderTest, DirectSolrConnectionTest, TestAuthorizationFramework, AnalysisAfterCoreReloadTest, SOLR749Test, XmlUpdateRequestHandlerTest, FileUtilsTest, TestSubQueryTransformerCrossCore, TestBinaryResponseWriter, AssignTest, OverseerTest, TestBM25SimilarityFactory, TestDefaultStatsCache, TestMiniSolrCloudClusterSSL, TestTolerantUpdateProcessorRandomCloud, TestFaceting, OverriddenZkACLAndCredentialsProvidersTest, UnloadDistributedZkTest, TestRTGBase, DocExpirationUpdateProcessorFactoryTest, QueryEqualityTest, TestRequestForwarding, StandardRequestHandlerTest, TestSweetSpotSimilarityFactory, PrimUtilsTest, TestHighlightDedupGrouping, PolyFieldTest, TestChildDocTransformer, CopyFieldTest, DistributedFacetPivotWhiteBoxTest, TestReloadDeadlock, TestDefaultSearchFieldResource, TestFieldResource, TestReplicationHandlerBackup, TestCollationField, TestFieldCacheReopen, CdcrUpdateLogTest, ExitableDirectoryReaderTest, TestHashPartitioner, LeaderInitiatedRecoveryOnCommitTest, SearchHandlerTest, RemoteQueryErrorTest, TestFileDictionaryLookup, ShardRoutingTest, CollectionsAPIAsyncDistributedZkTest, TestPhraseSuggestions, TestCSVResponseWriter, SuggesterFSTTest, TestExtendedDismaxParser, TestCSVLoader, CustomCollectionTest, QueryElevationComponentTest, TestPostingsSolrHighlighter, PeerSyncReplicationTest]
[junit4] Completed [238/680 (1!)] on J0 in 216.93s, 1 test, 1 failure <<< FAILURES!
[...truncated 63827 lines...]
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3801
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3801/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC
1 tests failed.
FAILED: org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit
Error Message:
expected:<1> but was:<2>
Stack Trace:
java.lang.AssertionError: expected:<1> but was:<2>
at __randomizedtesting.SeedInfo.seed([EE51548B2A6144F4:171CC7241614097E]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:283)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 10962 lines...]
[junit4] Suite: org.apache.solr.cloud.ShardSplitTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/init-core-data-001
[junit4] 2> 240058 INFO (SUITE-ShardSplitTest-seed#[EE51548B2A6144F4]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields
[junit4] 2> 240062 INFO (SUITE-ShardSplitTest-seed#[EE51548B2A6144F4]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 240062 INFO (SUITE-ShardSplitTest-seed#[EE51548B2A6144F4]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /pqmsy/
[junit4] 2> 240064 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 240065 INFO (Thread-384) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 240065 INFO (Thread-384) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 240170 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkTestServer start zk server on port:52694
[junit4] 2> 240205 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 240209 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 240213 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 240216 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 240219 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 240222 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 240225 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 240228 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 240232 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 240235 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 240238 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 240542 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/control-001/cores/collection1
[junit4] 2> 240544 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 240546 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@635d75fc{/pqmsy,null,AVAILABLE}
[junit4] 2> 240548 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3036c676{HTTP/1.1,[http/1.1]}{127.0.0.1:52697}
[junit4] 2> 240548 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server Started @248937ms
[junit4] 2> 240548 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/tempDir-001/control/data, hostContext=/pqmsy, hostPort=52697, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/control-001/cores}
[junit4] 2> 240548 ERROR (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 240548 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 240548 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 240548 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 240549 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-27T04:00:08.366Z
[junit4] 2> 240554 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 240554 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/control-001/solr.xml
[junit4] 2> 240566 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52694/solr
[junit4] 2> 240623 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52697_pqmsy
[junit4] 2> 240626 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.Overseer Overseer (id=97353046942285828-127.0.0.1:52697_pqmsy-n_0000000000) starting
[junit4] 2> 240647 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52697_pqmsy
[junit4] 2> 240652 INFO (OverseerStateUpdate-97353046942285828-127.0.0.1:52697_pqmsy-n_0000000000) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 240750 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/control-001/cores
[junit4] 2> 240750 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 240756 INFO (OverseerStateUpdate-97353046942285828-127.0.0.1:52697_pqmsy-n_0000000000) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 241788 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 241803 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 241927 WARN (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 241928 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 241971 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 241972 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/control-001/cores/collection1/data/]
[junit4] 2> 241972 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@da160ff
[junit4] 2> 241976 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=5.63671875, floorSegmentMB=0.2470703125, forceMergeDeletesPctAllowed=22.017257614259755, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8726594395281767
[junit4] 2> 241987 WARN (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 242006 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 242006 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 242008 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 242008 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 242009 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 242010 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@58833da0[collection1] main]
[junit4] 2> 242013 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 242014 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 242014 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 242016 INFO (searcherExecutor-935-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@58833da0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 242017 INFO (coreLoadExecutor-934-thread-1-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557648753121296384
[junit4] 2> 242034 INFO (coreZkRegister-927-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 242034 INFO (coreZkRegister-927-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 242034 INFO (coreZkRegister-927-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52697/pqmsy/collection1/
[junit4] 2> 242034 INFO (coreZkRegister-927-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 242034 INFO (coreZkRegister-927-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:52697/pqmsy/collection1/ has no replicas
[junit4] 2> 242044 INFO (coreZkRegister-927-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52697/pqmsy/collection1/ shard1
[junit4] 2> 242159 INFO (coreZkRegister-927-thread-1-processing-n:127.0.0.1:52697_pqmsy x:collection1 c:control_collection) [n:127.0.0.1:52697_pqmsy c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 242407 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 242410 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52694/solr ready
[junit4] 2> 242410 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 242410 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 242856 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-1-001/cores/collection1
[junit4] 2> 242858 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-1-001
[junit4] 2> 242859 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 242887 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7ff538ec{/pqmsy,null,AVAILABLE}
[junit4] 2> 242887 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2fd4f8ca{HTTP/1.1,[http/1.1]}{127.0.0.1:52707}
[junit4] 2> 242888 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server Started @251276ms
[junit4] 2> 242888 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/pqmsy, hostPort=52707, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-1-001/cores}
[junit4] 2> 242888 ERROR (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 242889 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 242889 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 242889 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 242889 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-27T04:00:10.706Z
[junit4] 2> 242897 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 242897 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-1-001/solr.xml
[junit4] 2> 242915 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52694/solr
[junit4] 2> 242940 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 242954 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52707_pqmsy
[junit4] 2> 242958 INFO (zkCallback-364-thread-2-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 242959 INFO (zkCallback-368-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 242959 INFO (zkCallback-374-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 243093 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-1-001/cores
[junit4] 2> 243093 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 243099 INFO (OverseerStateUpdate-97353046942285828-127.0.0.1:52697_pqmsy-n_0000000000) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 243213 INFO (zkCallback-374-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 244125 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 244157 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 244218 WARN (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 244220 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 244259 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 244259 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 244260 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@da160ff
[junit4] 2> 244262 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=5.63671875, floorSegmentMB=0.2470703125, forceMergeDeletesPctAllowed=22.017257614259755, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8726594395281767
[junit4] 2> 244285 WARN (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 244303 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 244304 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 244304 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 244304 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 244305 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 244306 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@384dcbe1[collection1] main]
[junit4] 2> 244309 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 244310 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 244310 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 244311 INFO (searcherExecutor-946-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@384dcbe1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 244312 INFO (coreLoadExecutor-945-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557648755527778304
[junit4] 2> 244325 INFO (coreZkRegister-940-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 244325 INFO (coreZkRegister-940-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 244325 INFO (coreZkRegister-940-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52707/pqmsy/collection1/
[junit4] 2> 244325 INFO (coreZkRegister-940-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 244325 INFO (coreZkRegister-940-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:52707/pqmsy/collection1/ has no replicas
[junit4] 2> 244335 INFO (coreZkRegister-940-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52707/pqmsy/collection1/ shard2
[junit4] 2> 244451 INFO (zkCallback-374-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 244505 INFO (coreZkRegister-940-thread-1-processing-n:127.0.0.1:52707_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 244625 INFO (zkCallback-374-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 245107 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-2-001/cores/collection1
[junit4] 2> 245110 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-2-001
[junit4] 2> 245114 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 245117 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@52ea8fd8{/pqmsy,null,AVAILABLE}
[junit4] 2> 245118 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@afccc17{HTTP/1.1,[http/1.1]}{127.0.0.1:52715}
[junit4] 2> 245118 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server Started @253507ms
[junit4] 2> 245118 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/pqmsy, hostPort=52715, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-2-001/cores}
[junit4] 2> 245118 ERROR (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 245124 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 245124 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 245125 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 245125 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-27T04:00:12.942Z
[junit4] 2> 245130 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 245130 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-2-001/solr.xml
[junit4] 2> 245148 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52694/solr
[junit4] 2> 245169 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 245179 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52715_pqmsy
[junit4] 2> 245185 INFO (zkCallback-368-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 245187 INFO (zkCallback-380-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 245186 INFO (zkCallback-364-thread-3-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 245185 INFO (zkCallback-374-thread-1-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 245261 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-2-001/cores
[junit4] 2> 245261 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 245265 INFO (OverseerStateUpdate-97353046942285828-127.0.0.1:52697_pqmsy-n_0000000000) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 245378 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 245378 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 246284 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 246325 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 246409 WARN (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 246411 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 246453 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 246453 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 246454 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@da160ff
[junit4] 2> 246459 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=5.63671875, floorSegmentMB=0.2470703125, forceMergeDeletesPctAllowed=22.017257614259755, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8726594395281767
[junit4] 2> 246474 WARN (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 246499 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 246499 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 246501 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 246501 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 246502 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 246504 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4970dade[collection1] main]
[junit4] 2> 246506 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 246507 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 246507 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 246509 INFO (searcherExecutor-957-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4970dade[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 246510 INFO (coreLoadExecutor-956-thread-1-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557648757832548352
[junit4] 2> 246524 INFO (coreZkRegister-951-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 246524 INFO (coreZkRegister-951-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 246524 INFO (coreZkRegister-951-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:52715/pqmsy/collection1/
[junit4] 2> 246524 INFO (coreZkRegister-951-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 246524 INFO (coreZkRegister-951-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:52715/pqmsy/collection1/ has no replicas
[junit4] 2> 246535 INFO (coreZkRegister-951-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:52715/pqmsy/collection1/ shard1
[junit4] 2> 246645 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 246645 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 246697 INFO (coreZkRegister-951-thread-1-processing-n:127.0.0.1:52715_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52715_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 246807 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 246807 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 247174 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-3-001/cores/collection1
[junit4] 2> 247177 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-3-001
[junit4] 2> 247177 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 247181 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@387f9911{/pqmsy,null,AVAILABLE}
[junit4] 2> 247182 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@43ac5d2e{HTTP/1.1,[http/1.1]}{127.0.0.1:52724}
[junit4] 2> 247182 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server Started @255570ms
[junit4] 2> 247182 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/pqmsy, hostPort=52724, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-3-001/cores}
[junit4] 2> 247182 ERROR (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 247183 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 247183 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 247184 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 247184 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-27T04:00:15.001Z
[junit4] 2> 247193 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 247193 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-3-001/solr.xml
[junit4] 2> 247225 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52694/solr
[junit4] 2> 247250 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 247262 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52724_pqmsy
[junit4] 2> 247266 INFO (zkCallback-364-thread-3-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 247267 INFO (zkCallback-386-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 247267 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 247266 INFO (zkCallback-368-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 247266 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 247307 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-3-001/cores
[junit4] 2> 247307 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 247322 INFO (OverseerStateUpdate-97353046942285828-127.0.0.1:52697_pqmsy-n_0000000000) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 247433 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 247433 INFO (zkCallback-386-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 247433 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 248344 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 248357 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 248450 WARN (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 248452 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 248490 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 248490 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 248490 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@da160ff
[junit4] 2> 248493 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=5.63671875, floorSegmentMB=0.2470703125, forceMergeDeletesPctAllowed=22.017257614259755, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8726594395281767
[junit4] 2> 248510 WARN (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 248531 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 248531 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 248532 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 248532 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 248534 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 248534 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@56dcf754[collection1] main]
[junit4] 2> 248536 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 248537 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 248537 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 248539 INFO (searcherExecutor-968-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@56dcf754[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 248540 INFO (coreLoadExecutor-967-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557648759961157632
[junit4] 2> 248547 INFO (coreZkRegister-962-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 248547 INFO (updateExecutor-383-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 248548 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 248548 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 248548 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 248548 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 248548 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:52707/pqmsy/collection1/] and I am [http://127.0.0.1:52724/pqmsy/collection1/]
[junit4] 2> 248552 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:52707/pqmsy]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:52724_pqmsy&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 248556 INFO (qtp407216533-1916) [n:127.0.0.1:52707_pqmsy ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 248557 INFO (qtp407216533-1916) [n:127.0.0.1:52707_pqmsy ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard2 of collection1) have state: recovering
[junit4] 2> 248557 INFO (qtp407216533-1916) [n:127.0.0.1:52707_pqmsy ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:52724_pqmsy, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:52724/pqmsy","node_name":"127.0.0.1:52724_pqmsy","state":"down"}
[junit4] 2> 248669 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 248669 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 248669 INFO (zkCallback-386-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 249198 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-4-001/cores/collection1
[junit4] 2> 249199 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-4-001
[junit4] 2> 249199 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 249201 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@e8f82a8{/pqmsy,null,AVAILABLE}
[junit4] 2> 249202 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@10b00fc1{HTTP/1.1,[http/1.1]}{127.0.0.1:52733}
[junit4] 2> 249202 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.Server Started @257590ms
[junit4] 2> 249202 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/pqmsy, hostPort=52733, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-4-001/cores}
[junit4] 2> 249203 ERROR (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 249203 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 249203 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 249203 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 249203 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-27T04:00:17.020Z
[junit4] 2> 249209 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 249209 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-4-001/solr.xml
[junit4] 2> 249237 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52694/solr
[junit4] 2> 249259 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52733_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 249270 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52733_pqmsy ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52733_pqmsy
[junit4] 2> 249277 INFO (zkCallback-386-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 249285 INFO (zkCallback-393-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 249284 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 249277 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 249277 INFO (zkCallback-364-thread-3-processing-n:127.0.0.1:52697_pqmsy) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 249277 INFO (zkCallback-368-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 249317 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52733_pqmsy ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-4-001/cores
[junit4] 2> 249317 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [n:127.0.0.1:52733_pqmsy ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 249324 INFO (OverseerStateUpdate-97353046942285828-127.0.0.1:52697_pqmsy-n_0000000000) [n:127.0.0.1:52697_pqmsy ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 249433 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 249433 INFO (zkCallback-393-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 249433 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 249433 INFO (zkCallback-386-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 249567 INFO (qtp407216533-1916) [n:127.0.0.1:52707_pqmsy ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:52724_pqmsy, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:52724/pqmsy","node_name":"127.0.0.1:52724_pqmsy","state":"recovering"}
[junit4] 2> 249567 INFO (qtp407216533-1916) [n:127.0.0.1:52707_pqmsy ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 249568 INFO (qtp407216533-1916) [n:127.0.0.1:52707_pqmsy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:52724_pqmsy&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1011
[junit4] 2> 250346 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 250359 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 250430 WARN (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 250432 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 250473 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 250473 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-4-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001/shard-4-001/cores/collection1/data/]
[junit4] 2> 250473 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@da160ff
[junit4] 2> 250476 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=5.63671875, floorSegmentMB=0.2470703125, forceMergeDeletesPctAllowed=22.017257614259755, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8726594395281767
[junit4] 2> 250484 WARN (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 250502 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 250502 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 250503 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 250503 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 250504 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 250505 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@13e03e7[collection1] main]
[junit4] 2> 250507 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 250507 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 250508 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 250509 INFO (searcherExecutor-979-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@13e03e7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 250509 INFO (coreLoadExecutor-978-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557648762025803776
[junit4] 2> 250516 INFO (coreZkRegister-973-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 c:collection1) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 250516 INFO (updateExecutor-390-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 250517 INFO (recoveryExecutor-391-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 250517 INFO (recoveryExecutor-391-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 250517 INFO (recoveryExecutor-391-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 250517 INFO (recoveryExecutor-391-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 250517 INFO (recoveryExecutor-391-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:52715/pqmsy/collection1/] and I am [http://127.0.0.1:52733/pqmsy/collection1/]
[junit4] 2> 250520 INFO (recoveryExecutor-391-thread-1-processing-n:127.0.0.1:52733_pqmsy x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:52733_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:52715/pqmsy]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:52733_pqmsy&coreNodeName=core_node4&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 250524 INFO (qtp1031597028-1944) [n:127.0.0.1:52715_pqmsy ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node4, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 250526 INFO (qtp1031597028-1944) [n:127.0.0.1:52715_pqmsy ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 250526 INFO (qtp1031597028-1944) [n:127.0.0.1:52715_pqmsy ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:52733_pqmsy, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{"core":"collection1","base_url":"http://127.0.0.1:52733/pqmsy","node_name":"127.0.0.1:52733_pqmsy","state":"down"}
[junit4] 2> 250638 INFO (zkCallback-380-thread-2-processing-n:127.0.0.1:52715_pqmsy) [n:127.0.0.1:52715_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 250638 INFO (zkCallback-393-thread-1-processing-n:127.0.0.1:52733_pqmsy) [n:127.0.0.1:52733_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 250638 INFO (zkCallback-374-thread-2-processing-n:127.0.0.1:52707_pqmsy) [n:127.0.0.1:52707_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 250638 INFO (zkCallback-386-thread-1-processing-n:127.0.0.1:52724_pqmsy) [n:127.0.0.1:52724_pqmsy ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [5])
[junit4] 2> 250928 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 250928 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 15 for each attempt
[junit4] 2> 250928 INFO (TEST-ShardSplitTest.test-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):15
[junit4] 2> 251527 INFO (qtp1031597028-1944) [n:127.0.0.1:52715_pqmsy ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:52733_pqmsy, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{"core":"collection1","base_url":"http://127.0.0.1:52733/pqmsy","node_name":"127.0.0.1:52733_pqmsy","state":"recovering"}
[junit4] 2> 251527 INFO (qtp1031597028-1944) [n:127.0.0.1:52715_pqmsy ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node4, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 251527 INFO (qtp1031597028-1944) [n:127.0.0.1:52715_pqmsy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:52733_pqmsy&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node4&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1003
[junit4] 2> 256569 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:52707/pqmsy/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 256569 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:52724/pqmsy START replicas=[http://127.0.0.1:52707/pqmsy/collection1/] nUpdates=100
[junit4] 2> 256575 INFO (qtp407216533-1920) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 256575 INFO (qtp407216533-1920) [n:127.0.0.1:52707_pqmsy c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp=/pqmsy path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 256577 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 256577 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:52724_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 256577 INFO (recoveryExecutor-384-thread-1-processing-n:127.0.0.1:52724_pqmsy x:collection1
[...truncated too long message...]
:testSplitStaticIndexReplication_shard1_0_replica1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testSplitStaticIndexReplication.shard1_0.replica1
[junit4] 2> 539330 INFO (recoveryExecutor-581-thread-1-processing-n:127.0.0.1:53301_pqmsy x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:53301_pqmsy c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 539333 WARN (zkCallback-583-thread-1-processing-n:127.0.0.1:53301_pqmsy) [n:127.0.0.1:53301_pqmsy ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 539334 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 539334 INFO (zkCallback-549-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 539334 INFO (zkCallback-573-thread-1-processing-n:127.0.0.1:53314_pqmsy) [n:127.0.0.1:53314_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 539335 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@2d1c06b6{HTTP/1.1,[http/1.1]}{127.0.0.1:53301}
[junit4] 2> 539335 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1a5d0cca{/pqmsy,null,UNAVAILABLE}
[junit4] 2> 539337 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 53306
[junit4] 2> 539337 INFO (zkCallback-566-thread-2-processing-n:127.0.0.1:53306_pqmsy) [n:127.0.0.1:53306_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 539337 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=879483284
[junit4] 2> 539342 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.Overseer Overseer (id=97353063404273678-127.0.0.1:53306_pqmsy-n_0000000003) closing
[junit4] 2> 539342 INFO (OverseerStateUpdate-97353063404273678-127.0.0.1:53306_pqmsy-n_0000000003) [n:127.0.0.1:53306_pqmsy ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:53306_pqmsy
[junit4] 2> 539346 INFO (zkCallback-549-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 539346 INFO (zkCallback-573-thread-1-processing-n:127.0.0.1:53314_pqmsy) [n:127.0.0.1:53314_pqmsy ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:53314_pqmsy
[junit4] 2> 539346 INFO (zkCallback-573-thread-2-processing-n:127.0.0.1:53314_pqmsy) [n:127.0.0.1:53314_pqmsy ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 539349 INFO (zkCallback-573-thread-1-processing-n:127.0.0.1:53314_pqmsy) [n:127.0.0.1:53314_pqmsy ] o.a.s.c.Overseer Overseer (id=97353063404273681-127.0.0.1:53314_pqmsy-n_0000000004) starting
[junit4] 2> 540856 WARN (zkCallback-566-thread-3-processing-n:127.0.0.1:53306_pqmsy) [n:127.0.0.1:53306_pqmsy ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 541573 WARN (zkCallback-566-thread-1-processing-n:127.0.0.1:53306_pqmsy) [n:127.0.0.1:53306_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.SyncStrategy Closed, skipping sync up.
[junit4] 2> 541574 INFO (zkCallback-566-thread-1-processing-n:127.0.0.1:53306_pqmsy) [n:127.0.0.1:53306_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 541574 INFO (zkCallback-566-thread-1-processing-n:127.0.0.1:53306_pqmsy) [n:127.0.0.1:53306_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@5a8cf277
[junit4] 2> 541624 INFO (zkCallback-566-thread-1-processing-n:127.0.0.1:53306_pqmsy) [n:127.0.0.1:53306_pqmsy c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 541625 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 541627 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@3e90c0e{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 541627 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@51894b1{/pqmsy,null,UNAVAILABLE}
[junit4] 2> 541628 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 53314
[junit4] 2> 541628 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=509049753
[junit4] 2> 541632 INFO (coreCloseExecutor-1596-thread-1) [n:127.0.0.1:53314_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@4379f09f
[junit4] 2> 541656 INFO (coreCloseExecutor-1596-thread-1) [n:127.0.0.1:53314_pqmsy c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
[junit4] 2> 541658 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.Overseer Overseer (id=97353063404273681-127.0.0.1:53314_pqmsy-n_0000000004) closing
[junit4] 2> 541658 INFO (OverseerStateUpdate-97353063404273681-127.0.0.1:53314_pqmsy-n_0000000004) [n:127.0.0.1:53314_pqmsy ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:53314_pqmsy
[junit4] 2> 541663 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 541664 INFO (zkCallback-549-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 541664 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@693bff5a{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 541665 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6a67600c{/pqmsy,null,UNAVAILABLE}
[junit4] 2> 541669 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 53282
[junit4] 2> 541670 INFO (TEST-ShardSplitTest.testSplitStaticIndexReplication-seed#[EE51548B2A6144F4]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53279 53279
[junit4] 2> 541685 INFO (Thread-638) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53279 53279
[junit4] 2> 543753 WARN (Thread-638) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/aliases.json
[junit4] 2> 8 /solr/clusterprops.json
[junit4] 2> 6 /solr/security.json
[junit4] 2> 6 /solr/configs/conf1
[junit4] 2> 5 /solr/collections/collection1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/clusterstate.json
[junit4] 2> 4 /solr/collections/testSplitStaticIndexReplication/state.json
[junit4] 2> 2 /solr/overseer_elect/election/97353063404273672-127.0.0.1:53295_pqmsy-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 87 /solr/overseer/queue
[junit4] 2> 38 /solr/overseer/collection-queue-work
[junit4] 2> 20 /solr/overseer/queue-work
[junit4] 2> 8 /solr/collections
[junit4] 2> 7 /solr/live_nodes
[junit4] 2>
[junit4] 2> 543754 INFO (SUITE-ShardSplitTest-seed#[EE51548B2A6144F4]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_EE51548B2A6144F4-001
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=false): {}, locale=sr-ME, timezone=America/Fort_Nelson
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=265749584,total=471130112
[junit4] 2> NOTE: All tests run in this JVM: [FieldMutatingUpdateProcessorTest, DistributedTermsComponentTest, TestOverriddenPrefixQueryForCustomFieldType, CSVRequestHandlerTest, RAMDirectoryFactoryTest, AutoCommitTest, TestLRUStatsCache, TestNRTOpen, TestAnalyzedSuggestions, TestSimpleQParserPlugin, TestDynamicLoading, TestMinMaxOnMultiValuedField, BlockDirectoryTest, HighlighterConfigTest, BlockJoinFacetSimpleTest, TestPHPSerializedResponseWriter, LoggingHandlerTest, FullSolrCloudDistribCmdsTest, SliceStateTest, ShowFileRequestHandlerTest, AliasIntegrationTest, SolrInfoMBeanTest, ShardSplitTest]
[junit4] Completed [89/684 (1!)] on J1 in 303.72s, 5 tests, 1 failure <<< FAILURES!
[...truncated 64363 lines...]
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3800
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3800/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
timeout waiting to see all nodes active
Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
at __randomizedtesting.SeedInfo.seed([A0E5E2CB84D8E147:28B1DD112A248CBF]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12286 lines...]
[junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/init-core-data-001
[junit4] 2> 2641237 INFO (SUITE-PeerSyncReplicationTest-seed#[A0E5E2CB84D8E147]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields
[junit4] 2> 2641241 INFO (SUITE-PeerSyncReplicationTest-seed#[A0E5E2CB84D8E147]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 2641242 INFO (SUITE-PeerSyncReplicationTest-seed#[A0E5E2CB84D8E147]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 2641244 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2641244 INFO (Thread-2880) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2641244 INFO (Thread-2880) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2641347 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ZkTestServer start zk server on port:52443
[junit4] 2> 2641380 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2641448 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 2641561 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2641650 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2641687 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2641712 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2641744 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2641794 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2641813 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2641830 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2641855 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2645831 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/control-001/cores/collection1
[junit4] 2> 2645859 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2645864 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7a9f2e46{/,null,AVAILABLE}
[junit4] 2> 2645879 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@8cd1f39{SSL,[ssl, http/1.1]}{127.0.0.1:52473}
[junit4] 2> 2645880 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server Started @2651354ms
[junit4] 2> 2645880 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/tempDir-001/control/data, hostContext=/, hostPort=52473, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/control-001/cores}
[junit4] 2> 2645880 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2645881 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2645881 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2645881 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2645881 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T19:57:03.824Z
[junit4] 2> 2645888 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2645888 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/control-001/solr.xml
[junit4] 2> 2645905 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52443/solr
[junit4] 2> 2645969 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52473_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52473_
[junit4] 2> 2645971 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52473_ ] o.a.s.c.Overseer Overseer (id=97351147139235845-127.0.0.1:52473_-n_0000000000) starting
[junit4] 2> 2645990 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52473_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52473_
[junit4] 2> 2645995 INFO (zkCallback-2962-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2646098 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52473_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/control-001/cores
[junit4] 2> 2646099 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52473_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2646108 INFO (OverseerStateUpdate-97351147139235845-127.0.0.1:52473_-n_0000000000) [n:127.0.0.1:52473_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2647138 WARN (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2647139 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2647169 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2647306 WARN (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2647308 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2647340 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 2647340 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/control-001/cores/collection1/data/]
[junit4] 2> 2647341 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d92275b
[junit4] 2> 2647346 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1672653504, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2647368 WARN (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2647388 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2647388 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2647390 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2647390 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2647390 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=55.8642578125, floorSegmentMB=0.7197265625, forceMergeDeletesPctAllowed=2.102139642005424, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2647391 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@692f0eb4[collection1] main]
[junit4] 2> 2647393 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2647394 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2647394 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2647396 INFO (searcherExecutor-6484-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@692f0eb4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2647396 INFO (coreLoadExecutor-6483-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557618360675467264
[junit4] 2> 2647411 INFO (coreZkRegister-6476-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2647411 INFO (coreZkRegister-6476-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2647411 INFO (coreZkRegister-6476-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:52473/collection1/
[junit4] 2> 2647412 INFO (coreZkRegister-6476-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2647412 INFO (coreZkRegister-6476-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:52473/collection1/ has no replicas
[junit4] 2> 2647422 INFO (coreZkRegister-6476-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:52473/collection1/ shard1
[junit4] 2> 2647595 INFO (coreZkRegister-6476-thread-1-processing-n:127.0.0.1:52473_ x:collection1 c:control_collection) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2647714 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2647717 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:52443/solr ready
[junit4] 2> 2647717 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 2648135 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-1-001/cores/collection1
[junit4] 2> 2648137 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-1-001
[junit4] 2> 2648142 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2648150 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2c2437e0{/,null,AVAILABLE}
[junit4] 2> 2648152 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4f40effb{SSL,[ssl, http/1.1]}{127.0.0.1:52478}
[junit4] 2> 2648152 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server Started @2653627ms
[junit4] 2> 2648152 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=52478, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-1-001/cores}
[junit4] 2> 2648152 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2648153 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2648153 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2648153 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2648153 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T19:57:06.096Z
[junit4] 2> 2648158 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2648158 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-1-001/solr.xml
[junit4] 2> 2648173 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52443/solr
[junit4] 2> 2648195 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52478_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2648219 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52478_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52478_
[junit4] 2> 2648224 INFO (zkCallback-2962-thread-1-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2648224 INFO (zkCallback-2966-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2648225 INFO (zkCallback-2971-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2648401 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52478_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-1-001/cores
[junit4] 2> 2648401 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52478_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2648405 INFO (OverseerStateUpdate-97351147139235845-127.0.0.1:52473_-n_0000000000) [n:127.0.0.1:52473_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2649426 WARN (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2649427 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2649444 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2649550 WARN (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2649560 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2649581 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2649581 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 2649581 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d92275b
[junit4] 2> 2649586 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1672653504, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2649595 WARN (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2649613 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2649613 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2649614 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2649614 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2649615 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=55.8642578125, floorSegmentMB=0.7197265625, forceMergeDeletesPctAllowed=2.102139642005424, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2649616 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@30ea41ba[collection1] main]
[junit4] 2> 2649618 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2649619 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2649619 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2649620 INFO (searcherExecutor-6495-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@30ea41ba[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2649622 INFO (coreLoadExecutor-6494-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557618363009597440
[junit4] 2> 2649637 INFO (coreZkRegister-6489-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2649637 INFO (coreZkRegister-6489-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2649637 INFO (coreZkRegister-6489-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:52478/collection1/
[junit4] 2> 2649637 INFO (coreZkRegister-6489-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2649638 INFO (coreZkRegister-6489-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:52478/collection1/ has no replicas
[junit4] 2> 2649652 INFO (coreZkRegister-6489-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:52478/collection1/ shard1
[junit4] 2> 2649816 INFO (coreZkRegister-6489-thread-1-processing-n:127.0.0.1:52478_ x:collection1 c:collection1) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2650264 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-2-001/cores/collection1
[junit4] 2> 2650265 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-2-001
[junit4] 2> 2650267 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2650270 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6819ca0{/,null,AVAILABLE}
[junit4] 2> 2650272 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@c974e59{SSL,[ssl, http/1.1]}{127.0.0.1:52496}
[junit4] 2> 2650272 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server Started @2655747ms
[junit4] 2> 2650273 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=52496, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-2-001/cores}
[junit4] 2> 2650274 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2650274 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2650274 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2650274 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2650274 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T19:57:08.217Z
[junit4] 2> 2650278 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2650279 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-2-001/solr.xml
[junit4] 2> 2650291 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52443/solr
[junit4] 2> 2650309 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52496_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2650319 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52496_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52496_
[junit4] 2> 2650322 INFO (zkCallback-2962-thread-3-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2650322 INFO (zkCallback-2966-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2650322 INFO (zkCallback-2971-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2650323 INFO (zkCallback-2977-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2650459 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52496_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-2-001/cores
[junit4] 2> 2650459 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52496_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2650463 INFO (OverseerStateUpdate-97351147139235845-127.0.0.1:52473_-n_0000000000) [n:127.0.0.1:52473_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2651484 WARN (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2651485 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2651498 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2651607 WARN (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2651610 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2651629 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2651630 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 2651630 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d92275b
[junit4] 2> 2651634 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1672653504, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2651672 WARN (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2651716 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2651716 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2651718 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2651718 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2651719 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=55.8642578125, floorSegmentMB=0.7197265625, forceMergeDeletesPctAllowed=2.102139642005424, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2651721 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@33ffcef2[collection1] main]
[junit4] 2> 2651723 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2651724 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2651724 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2651726 INFO (searcherExecutor-6506-thread-1-processing-n:127.0.0.1:52496_ x:collection1 c:collection1) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@33ffcef2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2651727 INFO (coreLoadExecutor-6505-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557618365216849920
[junit4] 2> 2651734 INFO (coreZkRegister-6500-thread-1-processing-n:127.0.0.1:52496_ x:collection1 c:collection1) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2651734 INFO (updateExecutor-2974-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2651735 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2651736 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2651736 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2651736 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2651736 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [https://127.0.0.1:52478/collection1/] and I am [https://127.0.0.1:52496/collection1/]
[junit4] 2> 2651740 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [https://127.0.0.1:52478]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:52496_&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2651788 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2651789 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 2651789 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:52496_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"https://127.0.0.1:52496","node_name":"127.0.0.1:52496_","state":"down"}
[junit4] 2> 2652471 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-3-001/cores/collection1
[junit4] 2> 2652472 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-3-001
[junit4] 2> 2652475 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2652478 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7985df93{/,null,AVAILABLE}
[junit4] 2> 2652478 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@29d19d84{SSL,[ssl, http/1.1]}{127.0.0.1:52506}
[junit4] 2> 2652478 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.e.j.s.Server Started @2657953ms
[junit4] 2> 2652480 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=52506, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-3-001/cores}
[junit4] 2> 2652481 ERROR (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2652481 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2652481 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2652481 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2652481 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T19:57:10.424Z
[junit4] 2> 2652486 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2652486 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-3-001/solr.xml
[junit4] 2> 2652514 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52443/solr
[junit4] 2> 2652533 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52506_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2652543 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52506_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52506_
[junit4] 2> 2652546 INFO (zkCallback-2971-thread-1-processing-n:127.0.0.1:52478_) [n:127.0.0.1:52478_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2652548 INFO (zkCallback-2984-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2652548 INFO (zkCallback-2977-thread-1-processing-n:127.0.0.1:52496_) [n:127.0.0.1:52496_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2652548 INFO (zkCallback-2962-thread-3-processing-n:127.0.0.1:52473_) [n:127.0.0.1:52473_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2652548 INFO (zkCallback-2966-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2652668 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52506_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-3-001/cores
[junit4] 2> 2652668 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [n:127.0.0.1:52506_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2652673 INFO (OverseerStateUpdate-97351147139235845-127.0.0.1:52473_-n_0000000000) [n:127.0.0.1:52473_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2652790 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:52496_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"https://127.0.0.1:52496","node_name":"127.0.0.1:52496_","state":"recovering"}
[junit4] 2> 2652790 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2652790 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:52496_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1002
[junit4] 2> 2653695 WARN (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 2653696 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2653716 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2653853 WARN (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2653855 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2653884 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2653884 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 2653885 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4d92275b
[junit4] 2> 2653888 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1672653504, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2653903 WARN (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2653928 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2653928 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2653929 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2653929 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2653931 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=55.8642578125, floorSegmentMB=0.7197265625, forceMergeDeletesPctAllowed=2.102139642005424, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2653932 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@721e0b05[collection1] main]
[junit4] 2> 2653934 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2653935 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2653935 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2653936 INFO (searcherExecutor-6517-thread-1-processing-n:127.0.0.1:52506_ x:collection1 c:collection1) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@721e0b05[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2653937 INFO (coreLoadExecutor-6516-thread-1-processing-n:127.0.0.1:52506_) [n:127.0.0.1:52506_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557618367534202880
[junit4] 2> 2653946 INFO (coreZkRegister-6511-thread-1-processing-n:127.0.0.1:52506_ x:collection1 c:collection1) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2653947 INFO (updateExecutor-2981-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2653948 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2653948 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2653949 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2653949 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2653949 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [https://127.0.0.1:52478/collection1/] and I am [https://127.0.0.1:52506/collection1/]
[junit4] 2> 2653952 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [https://127.0.0.1:52478]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:52506_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2653962 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2653963 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 2653963 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:52506_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"https://127.0.0.1:52506","node_name":"127.0.0.1:52506_","state":"down"}
[junit4] 2> 2654101 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2654101 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30 for each attempt
[junit4] 2> 2654101 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 2654971 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:52506_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"https://127.0.0.1:52506","node_name":"127.0.0.1:52506_","state":"recovering"}
[junit4] 2> 2654972 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2654972 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:52506_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1009
[junit4] 2> 2659793 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [https://127.0.0.1:52478/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2659793 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=https://127.0.0.1:52496 START replicas=[https://127.0.0.1:52478/collection1/] nUpdates=1000
[junit4] 2> 2659805 INFO (qtp339668053-15688) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2659805 INFO (qtp339668053-15688) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 2659807 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2659807 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2659807 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2659807 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2659808 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2659808 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2659808 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2659808 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2659808 INFO (recoveryExecutor-2975-thread-1-processing-n:127.0.0.1:52496_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2661973 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [https://127.0.0.1:52478/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2661973 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=https://127.0.0.1:52506 START replicas=[https://127.0.0.1:52478/collection1/] nUpdates=1000
[junit4] 2> 2661980 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2661980 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2661982 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2661982 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2661982 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2661982 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2661983 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2661983 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2661983 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2661983 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2661983 INFO (recoveryExecutor-2982-thread-1-processing-n:127.0.0.1:52506_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2662115 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 2662144 INFO (qtp846221081-15648) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2662144 INFO (qtp846221081-15648) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2662145 INFO (qtp846221081-15648) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2662145 INFO (qtp846221081-15648) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 1
[junit4] 2> 2662163 INFO (qtp339668053-15687) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2662163 INFO (qtp339668053-15687) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2662164 INFO (qtp339668053-15687) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2662164 INFO (qtp339668053-15687) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:52478/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 2662185 INFO (qtp463464142-15713) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2662185 INFO (qtp463464142-15713) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2662185 INFO (qtp878465336-15748) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2662186 INFO (qtp878465336-15748) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2662186 INFO (qtp463464142-15713) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2662186 INFO (qtp878465336-15748) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2662186 INFO (qtp878465336-15748) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:52478/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 2662186 INFO (qtp463464142-15713) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:52478/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 2662186 INFO (qtp339668053-15686) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 35
[junit4] 2> 2662194 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2662201 INFO (qtp463464142-15716) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2662210 INFO (qtp878465336-15751) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.S.Request [collection1] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2664218 INFO (qtp846221081-15649) [n:127.0.0.1:52473_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557618378312515584)} 0 3
[junit4] 2> 2664227 INFO (qtp463464142-15709) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1557618378316709888&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557618378316709888)} 0 2
[junit4] 2> 2664227 INFO (qtp878465336-15744) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1557618378316709888&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557618378316709888)} 0 2
[junit4] 2> 2664227 INFO (qtp339668053-15684) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557618378316709888)} 0 7
[junit4] 2> 2664238 INFO (qtp878465336-15746) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[0 (1557618378330341376)]} 0 2
[junit4] 2> 2664238 INFO (qtp463464142-15711) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[0 (1557618378330341376)]} 0 3
[junit4] 2> 2664239 INFO (qtp339668053-15688) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[0 (1557618378330341376)]} 0 6
[junit4] 2> 2664241 INFO (qtp463464142-15712) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[1 (1557618378337681408)]} 0 0
[junit4] 2> 2664241 INFO (qtp878465336-15747) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[1 (1557618378337681408)]} 0 0
[junit4] 2> 2664242 INFO (qtp339668053-15687) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[1 (1557618378337681408)]} 0 2
[junit4] 2> 2664243 INFO (qtp463464142-15713) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[2 (1557618378339778560)]} 0 0
[junit4] 2> 2664243 INFO (qtp878465336-15748) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[2 (1557618378339778560)]} 0 0
[junit4] 2> 2664244 INFO (qtp339668053-15686) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[2 (1557618378339778560)]} 0 1
[junit4] 2> 2664253 INFO (qtp878465336-15750) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[3 (1557618378342924288)]} 0 0
[junit4] 2> 2664253 INFO (qtp463464142-15715) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[3 (1557618378342924288)]} 0 2
[junit4] 2> 2664254 INFO (qtp339668053-15681) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[3 (1557618378342924288)]} 0 8
[junit4] 2> 2664256 INFO (qtp463464142-15714) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[4 (1557618378353410048)]} 0 0
[junit4] 2> 2664256 INFO (qtp878465336-15750) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[4 (1557618378353410048)]} 0 0
[junit4] 2> 2664258 INFO (qtp339668053-15683) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[4 (1557618378353410048)]} 0 3
[junit4] 2> 2664260 INFO (qtp463464142-15716) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[5 (1557618378357604352)]} 0 0
[junit4] 2> 2664262 INFO (qtp878465336-15751) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[5 (1557618378357604352)]} 0 0
[junit4] 2> 2664263 INFO (qtp339668053-15685) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[5 (1557618378357604352)]} 0 4
[junit4] 2> 2664265 INFO (qtp878465336-15751) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[6 (1557618378362847232)]} 0 0
[junit4] 2> 2664266 INFO (qtp463464142-15716) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[6 (1557618378362847232)]} 0 0
[junit4] 2> 2664266 INFO (qtp339668053-15684) [n:127.0.0.1:52478_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{add=[6 (1557618378362847232)]} 0 2
[junit4] 2> 2664269 INFO (qtp463464142-15711) [n:127.0.0.1:52496_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:52478/collection1/&wt=javabin&version=2}{add=[7 (1557618378365992960)]} 0 0
[junit4] 2> 2664270 INFO (qtp878465336-15751) [n:127.0.0.1:52506_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= pa
[...truncated too long message...]
2855786 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ChaosMonkey monkey: stop shard! 52506
[junit4] 2> 2855788 INFO (TEST-PeerSyncReplicationTest.test-seed#[A0E5E2CB84D8E147]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52443 52443
[junit4] 2> 2855805 INFO (Thread-2880) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52443 52443
[junit4] 2> 2860650 WARN (Thread-2880) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 2 /solr/overseer_elect/election/97351147139235849-127.0.0.1:52478_-n_0000000001
[junit4] 2> 2 /solr/collections/collection1/leader_elect/shard1/election/97351147139235849-core_node1-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 209 /solr/overseer/collection-queue-work
[junit4] 2> 46 /solr/overseer/queue
[junit4] 2> 6 /solr/collections
[junit4] 2> 6 /solr/overseer/queue-work
[junit4] 2> 5 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=PeerSyncReplicationTest -Dtests.method=test -Dtests.seed=A0E5E2CB84D8E147 -Dtests.slow=true -Dtests.locale=nn-NO -Dtests.timezone=Australia/Broken_Hill -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] FAILURE 219s J1 | PeerSyncReplicationTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: timeout waiting to see all nodes active
[junit4] > at __randomizedtesting.SeedInfo.seed([A0E5E2CB84D8E147:28B1DD112A248CBF]:0)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2860694 INFO (SUITE-PeerSyncReplicationTest-seed#[A0E5E2CB84D8E147]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_A0E5E2CB84D8E147-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {other_tl1=PostingsFormat(name=Memory), range_facet_l_dv=FSTOrd50, rnd_s=PostingsFormat(name=LuceneVarGapDocFreqInterval), multiDefault=PostingsFormat(name=LuceneVarGapDocFreqInterval), a_t=PostingsFormat(name=Memory), intDefault=PostingsFormat(name=Memory), a_i1=PostingsFormat(name=Memory), id=FSTOrd50, range_facet_i_dv=PostingsFormat(name=Memory), text=Lucene50(blocksize=128), range_facet_l=PostingsFormat(name=Memory), timestamp=PostingsFormat(name=Memory)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene70), _version_=DocValuesFormat(name=Memory), range_facet_i_dv=DocValuesFormat(name=Memory), intDvoDefault=DocValuesFormat(name=Direct), timestamp=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1706, maxMBSortInHeap=5.210213964200542, sim=RandomSimilarity(queryNorm=false): {}, locale=nn-NO, timezone=Australia/Broken_Hill
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=224744368,total=530579456
[junit4] 2> NOTE: All tests run in this JVM: [TestHdfsUpdateLog, TemplateUpdateProcessorTest, TestOverriddenPrefixQueryForCustomFieldType, TestSolr4Spatial, ExternalFileFieldSortTest, ReturnFieldsTest, SuggesterWFSTTest, ResponseLogComponentTest, TestManagedResource, InfoHandlerTest, PreAnalyzedUpdateProcessorTest, TestOnReconnectListenerSupport, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestPHPSerializedResponseWriter, DateRangeFieldTest, BlockJoinFacetDistribTest, TestDistributedMissingSort, TestDocBasedVersionConstraints, BooleanFieldTest, TestMinMaxOnMultiValuedField, TestLuceneMatchVersion, TestBinaryResponseWriter, TestFoldingMultitermQuery, TestJsonFacetRefinement, TestSolrCloudWithKerberosAlt, HdfsUnloadDistributedZkTest, TestDistributedGrouping, MissingSegmentRecoveryTest, TestExclusionRuleCollectionAccess, TestOmitPositions, LeaderInitiatedRecoveryOnShardRestartTest, TestStressUserVersions, HighlighterConfigTest, SSLMigrationTest, StatsReloadRaceTest, TestSolrQueryParser, SortSpecParsingTest, MinimalSchemaTest, RequestHandlersTest, TestReversedWildcardFilterFactory, TestSortByMinMaxFunction, TestDelegationWithHadoopAuth, TestCrossCoreJoin, ConfigSetsAPITest, SolrJmxReporterTest, TestSuggestSpellingConverter, SolrCoreMetricManagerTest, TestRestManager, RegexBoostProcessorTest, LukeRequestHandlerTest, QueryResultKeyTest, TolerantUpdateProcessorTest, SignatureUpdateProcessorFactoryTest, TestEmbeddedSolrServerConstructors, OverseerTaskQueueTest, TestExportWriter, TestWriterPerf, TestFieldCacheSort, TestDefaultStatsCache, TestSizeLimitedDistributedMap, DistributedFacetPivotLargeTest, ShowFileRequestHandlerTest, CloneFieldUpdateProcessorFactoryTest, TestCodecSupport, CollectionReloadTest, TestCorePropertiesReload, TestHashQParserPlugin, TestCollectionAPI, TermVectorComponentDistributedTest, TestFiltering, TestGraphMLResponseWriter, TestRuleBasedAuthorizationPlugin, MergeStrategyTest, TestFilteredDocIdSet, TestFieldCacheVsDocValues, TestSimpleQParserPlugin, TestSegmentSorting, ClassificationUpdateProcessorIntegrationTest, TestRTimerTree, DebugComponentTest, BasicAuthStandaloneTest, AutoCommitTest, QueryParsingTest, TestReplicaProperties, AddBlockUpdateTest, TestConfigSetsAPIExclusivity, DistributedQueryComponentOptimizationTest, SolrXmlInZkTest, TestSearchPerf, JavabinLoaderTest, URLClassifyProcessorTest, RulesTest, ChaosMonkeySafeLeaderTest, TestSolrDynamicMBean, BlockDirectoryTest, TriLevelCompositeIdRoutingTest, TestReloadDeadlock, JsonLoaderTest, TestCustomSort, TestJsonRequest, TestPushWriter, TestInPlaceUpdatesStandalone, TestRestoreCore, TestPointFields, UUIDFieldTest, HdfsBasicDistributedZk2Test, TestExactSharedStatsCache, TestFuzzyAnalyzedSuggestions, SpellPossibilityIteratorTest, FullHLLTest, TestFieldCacheReopen, TestLegacyNumericRangeQueryBuilder, DistributedFacetPivotSmallAdvancedTest, NotRequiredUniqueKeyTest, TestCryptoKeys, HdfsRecoverLeaseTest, TestStressLiveNodes, HdfsThreadLeakTest, SuggestComponentContextFilterQueryTest, UpdateParamsTest, JSONWriterTest, ScriptEngineTest, DistribJoinFromCollectionTest, TestSimpleTrackingShardHandler, TestLMJelinekMercerSimilarityFactory, TestRandomCollapseQParserPlugin, HLLUtilTest, RecoveryAfterSoftCommitTest, TestAuthorizationFramework, DirectUpdateHandlerOptimizeTest, TestSchemaVersionResource, OpenCloseCoreStressTest, TestJmxMonitoredMap, ForceLeaderTest, TestCloudDeleteByQuery, CollectionTooManyReplicasTest, TestStressCloudBlindAtomicUpdates, DeleteNodeTest, IndexSchemaTest, TestSolrFieldCacheMBean, TestStressRecovery, FileBasedSpellCheckerTest, TestShardHandlerFactory, TestDistribDocBasedVersion, TestConfigSetProperties, SimpleMLTQParserTest, AliasIntegrationTest, StressHdfsTest, TestSubQueryTransformerCrossCore, FieldAnalysisRequestHandlerTest, TestAnalyzedSuggestions, TestSolrCoreProperties, DistribCursorPagingTest, SpellCheckCollatorTest, TestSolrDeletionPolicy2, BadIndexSchemaTest, TestHighFrequencyDictionaryFactory, TestTestInjection, ResourceLoaderTest, CircularListTest, CdcrReplicationHandlerTest, CdcrVersionReplicationTest, BigEndianAscendingWordDeserializerTest, TestSolrConfigHandlerConcurrent, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, LeaderElectionIntegrationTest, ClusterStateUpdateTest, TestRandomFaceting, ShardRoutingCustomTest, TestDistributedSearch, ZkControllerTest, TestStressReorder, TestJoin, TestStressVersions, HardAutoCommitTest, TestRangeQuery, SuggesterTSTTest, SuggesterTest, TestTrie, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, SolrCoreCheckLockOnStartupTest, FieldMutatingUpdateProcessorTest, TestAtomicUpdateErrorCases, DefaultValueUpdateProcessorTest, SortByFunctionTest, TestRemoteStreaming, DocValuesMultiTest, DistanceFunctionTest, SolrInfoMBeanTest, TestQueryUtils, DirectSolrSpellCheckerTest, PrimitiveFieldTypeTest, XmlUpdateRequestHandlerTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, FastVectorHighlighterTest, LoggingHandlerTest, TestCollationField, TestCSVResponseWriter, BinaryUpdateRequestHandlerTest, PingRequestHandlerTest, TestPhraseSuggestions, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestLMDirichletSimilarityFactory, TestFastWriter, TestFastLRUCache, PreAnalyzedFieldTest, DateFieldTest, ClusterStateTest, TestUtils, TestRTGBase, CursorPagingTest, CdcrRequestHandlerTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionStateFormat2Test, CollectionsAPISolrJTest, DeleteInactiveReplicaTest, DistributedQueueTest, DocValuesNotIndexedTest, HttpPartitionTest, OverseerCollectionConfigSetProcessorTest, OverseerRolesTest, PeerSyncReplicationTest]
[junit4] Completed [506/684 (1!)] on J1 in 219.49s, 1 test, 1 failure <<< FAILURES!
[...truncated 63041 lines...]
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3799
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3799/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
1 tests failed.
FAILED: org.apache.solr.cloud.PeerSyncReplicationTest.test
Error Message:
timeout waiting to see all nodes active
Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
at __randomizedtesting.SeedInfo.seed([ECCB97A5D970679F:649FA87F778C0A67]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12315 lines...]
[junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/init-core-data-001
[junit4] 2> 2687799 INFO (SUITE-PeerSyncReplicationTest-seed#[ECCB97A5D970679F]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields
[junit4] 2> 2687802 INFO (SUITE-PeerSyncReplicationTest-seed#[ECCB97A5D970679F]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 2687802 INFO (SUITE-PeerSyncReplicationTest-seed#[ECCB97A5D970679F]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /i/d
[junit4] 2> 2687807 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2687808 INFO (Thread-3425) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2687808 INFO (Thread-3425) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2687910 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.ZkTestServer start zk server on port:62260
[junit4] 2> 2687941 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 2687957 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 2687962 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 2687966 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 2687970 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 2687973 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 2687976 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 2687980 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 2687983 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 2687986 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 2687990 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 2689690 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/control-001/cores/collection1
[junit4] 2> 2689693 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2689696 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@163a0918{/i/d,null,AVAILABLE}
[junit4] 2> 2689696 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2d1ad56a{HTTP/1.1,[http/1.1]}{127.0.0.1:62263}
[junit4] 2> 2689696 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server Started @2695624ms
[junit4] 2> 2689697 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/tempDir-001/control/data, hostContext=/i/d, hostPort=62263, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/control-001/cores}
[junit4] 2> 2689697 ERROR (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2689698 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2689698 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2689698 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2689698 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T09:27:36.925Z
[junit4] 2> 2689703 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2689703 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/control-001/solr.xml
[junit4] 2> 2689730 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:62260/solr
[junit4] 2> 2689779 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:62263_i%2Fd
[junit4] 2> 2689781 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.Overseer Overseer (id=97348672227704836-127.0.0.1:62263_i%2Fd-n_0000000000) starting
[junit4] 2> 2689795 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62263_i%2Fd
[junit4] 2> 2689798 INFO (OverseerStateUpdate-97348672227704836-127.0.0.1:62263_i%2Fd-n_0000000000) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2689839 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/control-001/cores
[junit4] 2> 2689839 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2689844 INFO (OverseerStateUpdate-97348672227704836-127.0.0.1:62263_i%2Fd-n_0000000000) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2690861 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2690878 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2690967 WARN (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2690969 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2690989 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 2690989 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/control-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/control-001/cores/collection1/data/]
[junit4] 2> 2690989 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@9f078b
[junit4] 2> 2690994 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=2065311905, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2691038 WARN (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2691057 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2691058 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2691059 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2691059 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2691059 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=5.5322265625, floorSegmentMB=1.2119140625, forceMergeDeletesPctAllowed=3.699016373460373, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5180719917556548
[junit4] 2> 2691060 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5db329f0[collection1] main]
[junit4] 2> 2691062 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2691063 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2691063 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2691064 INFO (searcherExecutor-7724-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5db329f0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2691065 INFO (coreLoadExecutor-7723-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557578759056392192
[junit4] 2> 2691075 INFO (coreZkRegister-7716-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2691075 INFO (coreZkRegister-7716-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2691075 INFO (coreZkRegister-7716-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:62263/i/d/collection1/
[junit4] 2> 2691075 INFO (coreZkRegister-7716-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2691075 INFO (coreZkRegister-7716-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:62263/i/d/collection1/ has no replicas
[junit4] 2> 2691083 INFO (coreZkRegister-7716-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:62263/i/d/collection1/ shard1
[junit4] 2> 2691118 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2691120 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:62260/solr ready
[junit4] 2> 2691120 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 2691120 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 2691136 INFO (coreZkRegister-7716-thread-1-processing-n:127.0.0.1:62263_i%2Fd x:collection1 c:control_collection) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2691546 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-1-001/cores/collection1
[junit4] 2> 2691547 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-1-001
[junit4] 2> 2691548 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2691550 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3da55be9{/i/d,null,AVAILABLE}
[junit4] 2> 2691550 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3ab552af{HTTP/1.1,[http/1.1]}{127.0.0.1:62269}
[junit4] 2> 2691550 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server Started @2697479ms
[junit4] 2> 2691550 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/i/d, hostPort=62269, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-1-001/cores}
[junit4] 2> 2691552 ERROR (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2691552 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2691552 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2691552 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2691552 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T09:27:38.779Z
[junit4] 2> 2691556 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2691556 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-1-001/solr.xml
[junit4] 2> 2691624 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:62260/solr
[junit4] 2> 2691640 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2691650 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62269_i%2Fd
[junit4] 2> 2691653 INFO (zkCallback-3132-thread-3-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2691653 INFO (zkCallback-3136-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2691654 INFO (zkCallback-3142-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2691681 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-1-001/cores
[junit4] 2> 2691682 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2691685 INFO (OverseerStateUpdate-97348672227704836-127.0.0.1:62263_i%2Fd-n_0000000000) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2691795 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2692710 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2692726 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2692800 WARN (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2692802 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2692822 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2692822 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-1-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-1-001/cores/collection1/data/]
[junit4] 2> 2692822 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@9f078b
[junit4] 2> 2692836 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=2065311905, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2692859 WARN (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2692879 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2692879 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2692880 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2692880 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2692881 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=5.5322265625, floorSegmentMB=1.2119140625, forceMergeDeletesPctAllowed=3.699016373460373, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5180719917556548
[junit4] 2> 2692882 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@a53aadd[collection1] main]
[junit4] 2> 2692883 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2692884 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2692884 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2692885 INFO (searcherExecutor-7735-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@a53aadd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2692886 INFO (coreLoadExecutor-7734-thread-1-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557578760965849088
[junit4] 2> 2692896 INFO (coreZkRegister-7729-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2692897 INFO (coreZkRegister-7729-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2692897 INFO (coreZkRegister-7729-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:62269/i/d/collection1/
[junit4] 2> 2692897 INFO (coreZkRegister-7729-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2692897 INFO (coreZkRegister-7729-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:62269/i/d/collection1/ has no replicas
[junit4] 2> 2692904 INFO (coreZkRegister-7729-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:62269/i/d/collection1/ shard1
[junit4] 2> 2693016 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2693066 INFO (coreZkRegister-7729-thread-1-processing-n:127.0.0.1:62269_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2693174 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2693278 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-2-001/cores/collection1
[junit4] 2> 2693280 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-2-001
[junit4] 2> 2693281 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2693283 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@43204441{/i/d,null,AVAILABLE}
[junit4] 2> 2693283 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2673b2f3{HTTP/1.1,[http/1.1]}{127.0.0.1:62276}
[junit4] 2> 2693283 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server Started @2699212ms
[junit4] 2> 2693283 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/i/d, hostPort=62276, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-2-001/cores}
[junit4] 2> 2693284 ERROR (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2693284 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2693284 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2693284 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2693284 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T09:27:40.511Z
[junit4] 2> 2693290 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2693290 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-2-001/solr.xml
[junit4] 2> 2693310 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:62260/solr
[junit4] 2> 2693329 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2693339 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62276_i%2Fd
[junit4] 2> 2693352 INFO (zkCallback-3136-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2693353 INFO (zkCallback-3132-thread-1-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2693354 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2693354 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2693385 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-2-001/cores
[junit4] 2> 2693385 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2693390 INFO (OverseerStateUpdate-97348672227704836-127.0.0.1:62263_i%2Fd-n_0000000000) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2693495 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2693495 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2694403 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2694421 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2694498 WARN (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2694499 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2694519 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2694520 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-2-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-2-001/cores/collection1/data/]
[junit4] 2> 2694520 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@9f078b
[junit4] 2> 2694523 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=2065311905, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2694534 WARN (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2694555 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2694555 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2694556 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2694556 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2694557 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=5.5322265625, floorSegmentMB=1.2119140625, forceMergeDeletesPctAllowed=3.699016373460373, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5180719917556548
[junit4] 2> 2694557 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3a7267f[collection1] main]
[junit4] 2> 2694559 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2694560 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2694560 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2694561 INFO (searcherExecutor-7746-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3a7267f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2694562 INFO (coreLoadExecutor-7745-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557578762723262464
[junit4] 2> 2694568 INFO (coreZkRegister-7740-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2694568 INFO (updateExecutor-3145-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2694568 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2694568 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2694569 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2694569 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2694569 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:62269/i/d/collection1/] and I am [http://127.0.0.1:62276/i/d/collection1/]
[junit4] 2> 2694571 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:62269/i/d]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:62276_i%252Fd&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2694575 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2694575 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 2694575 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:62276_i%2Fd, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:62276/i/d","node_name":"127.0.0.1:62276_i%2Fd","state":"down"}
[junit4] 2> 2694686 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2694686 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 2695019 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-3-001/cores/collection1
[junit4] 2> 2695020 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-3-001
[junit4] 2> 2695021 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2695023 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1a894007{/i/d,null,AVAILABLE}
[junit4] 2> 2695023 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@41112563{HTTP/1.1,[http/1.1]}{127.0.0.1:62286}
[junit4] 2> 2695023 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.e.j.s.Server Started @2700952ms
[junit4] 2> 2695023 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/i/d, hostPort=62286, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-3-001/cores}
[junit4] 2> 2695024 ERROR (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2695025 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 2695025 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2695025 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2695025 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-26T09:27:42.252Z
[junit4] 2> 2695029 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 2695030 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-3-001/solr.xml
[junit4] 2> 2695051 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:62260/solr
[junit4] 2> 2695068 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 2695077 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62286_i%2Fd
[junit4] 2> 2695080 INFO (zkCallback-3136-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2695080 INFO (zkCallback-3132-thread-3-processing-n:127.0.0.1:62263_i%2Fd) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2695080 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2695081 INFO (zkCallback-3155-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2695081 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2695145 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-3-001/cores
[junit4] 2> 2695145 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2695148 INFO (OverseerStateUpdate-97348672227704836-127.0.0.1:62263_i%2Fd-n_0000000000) [n:127.0.0.1:62263_i%2Fd ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 2695254 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2695254 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2695254 INFO (zkCallback-3155-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2695577 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:62276_i%2Fd, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:62276/i/d","node_name":"127.0.0.1:62276_i%2Fd","state":"recovering"}
[junit4] 2> 2695578 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2695578 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:62276_i%252Fd&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1002
[junit4] 2> 2696170 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2696188 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2696263 WARN (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 2696265 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2696284 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
[junit4] 2> 2696284 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-3-001/cores/collection1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001/shard-3-001/cores/collection1/data/]
[junit4] 2> 2696284 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@9f078b
[junit4] 2> 2696288 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=2065311905, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2696316 WARN (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 2696334 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2696334 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2696335 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2696335 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2696337 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=5.5322265625, floorSegmentMB=1.2119140625, forceMergeDeletesPctAllowed=3.699016373460373, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5180719917556548
[junit4] 2> 2696338 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@ad1b8a5[collection1] main]
[junit4] 2> 2696339 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2696340 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2696340 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2696341 INFO (searcherExecutor-7757-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@ad1b8a5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2696342 INFO (coreLoadExecutor-7756-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557578764589727744
[junit4] 2> 2696348 INFO (coreZkRegister-7751-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 c:collection1) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
[junit4] 2> 2696348 INFO (updateExecutor-3152-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 2696349 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 2696349 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 2696349 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
[junit4] 2> 2696349 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 2696349 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:62269/i/d/collection1/] and I am [http://127.0.0.1:62286/i/d/collection1/]
[junit4] 2> 2696352 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:62269/i/d]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:62286_i%252Fd&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 2696353 INFO (qtp1543301162-17464) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 2696354 INFO (qtp1543301162-17464) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
[junit4] 2> 2696354 INFO (qtp1543301162-17464) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:62286_i%2Fd, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:62286/i/d","node_name":"127.0.0.1:62286_i%2Fd","state":"down"}
[junit4] 2> 2696455 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2696456 INFO (zkCallback-3155-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2696455 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2696506 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2696506 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30 for each attempt
[junit4] 2> 2696506 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30
[junit4] 2> 2697354 INFO (qtp1543301162-17464) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:62286_i%2Fd, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"http://127.0.0.1:62286/i/d","node_name":"127.0.0.1:62286_i%2Fd","state":"recovering"}
[junit4] 2> 2697354 INFO (qtp1543301162-17464) [n:127.0.0.1:62269_i%2Fd ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 2697354 INFO (qtp1543301162-17464) [n:127.0.0.1:62269_i%2Fd ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:62286_i%252Fd&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1001
[junit4] 2> 2702584 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:62269/i/d/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2702585 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:62276/i/d START replicas=[http://127.0.0.1:62269/i/d/collection1/] nUpdates=1000
[junit4] 2> 2702587 INFO (qtp1543301162-17461) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2702587 INFO (qtp1543301162-17461) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp=/i/d path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2702588 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2702588 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2702588 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2702588 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2702589 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2702589 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2702589 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2702589 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2702589 INFO (recoveryExecutor-3146-thread-1-processing-n:127.0.0.1:62276_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2702594 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2702594 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2702594 INFO (zkCallback-3155-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2704359 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:62269/i/d/collection1/] - recoveringAfterStartup=[true]
[junit4] 2> 2704360 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync PeerSync: core=collection1 url=http://127.0.0.1:62286/i/d START replicas=[http://127.0.0.1:62269/i/d/collection1/] nUpdates=1000
[junit4] 2> 2704362 INFO (qtp1543301162-17462) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2704362 INFO (qtp1543301162-17462) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp=/i/d path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2704363 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 2704363 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 2704363 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2704363 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2704363 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2704364 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 2704364 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[junit4] 2> 2704364 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 2704364 INFO (recoveryExecutor-3153-thread-1-processing-n:127.0.0.1:62286_i%2Fd x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 2704375 INFO (zkCallback-3142-thread-2-processing-n:127.0.0.1:62269_i%2Fd) [n:127.0.0.1:62269_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2704375 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:62276_i%2Fd) [n:127.0.0.1:62276_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2704375 INFO (zkCallback-3155-thread-1-processing-n:127.0.0.1:62286_i%2Fd) [n:127.0.0.1:62286_i%2Fd ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2704539 INFO (TEST-PeerSyncReplicationTest.test-seed#[ECCB97A5D970679F]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 2704542 INFO (qtp1875057789-17421) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2704542 INFO (qtp1875057789-17421) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2704543 INFO (qtp1875057789-17421) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2704544 INFO (qtp1875057789-17421) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/i/d path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 1
[junit4] 2> 2704547 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2704547 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2704549 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2704550 INFO (qtp1543301162-17460) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/i/d path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62269/i/d/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 2704553 INFO (qtp515180737-17526) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2704553 INFO (qtp515180737-17526) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2704553 INFO (qtp937108718-17491) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2704554 INFO (qtp937108718-17491) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2704555 INFO (qtp515180737-17526) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2704555 INFO (qtp515180737-17526) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/i/d path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62269/i/d/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 5
[junit4] 2> 2704555 INFO (qtp937108718-17491) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2704555 INFO (qtp937108718-17491) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/i/d path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:62269/i/d/collection1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 2704556 INFO (qtp1543301162-17465) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/i/d path=/update params={_stateVer_=collection1:9&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 10
[junit4] 2> 2704559 INFO (qtp1543301162-17461) [n:127.0.0.1:62269_i%2Fd c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.S.Request [collection1] webapp=/i/d path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2704561 INFO (qtp937108718-17493) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.S.Request [collection1] webapp=/i/d path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2704562 INFO (qtp515180737-17528) [n:127.0.0.1:62286_i%2Fd c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.S.Request [collection1] webapp=/i/d path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2706572 INFO (qtp1875057789-17422) [n:127.0.0.1:62263_i%2Fd c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/i/d path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557578775313514496)} 0 2
[junit4] 2> 2706577 INFO (qtp937108718-17494) [n:127.0.0.1:62276_i%2Fd c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/i/d path=/update params={
[...truncated too long message...]
2 WARN (Thread-3425) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 5 /solr/security.json
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 4 /solr/collections/collection1/state.json
[junit4] 2> 2 /solr/collections/collection1/leader_elect/shard1/election/97348672227704841-core_node1-n_0000000000
[junit4] 2> 2 /solr/overseer_elect/election/97348672227704841-127.0.0.1:62269_i%2Fd-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 207 /solr/overseer/collection-queue-work
[junit4] 2> 43 /solr/overseer/queue
[junit4] 2> 14 /solr/overseer/queue-work
[junit4] 2> 6 /solr/collections
[junit4] 2> 5 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=PeerSyncReplicationTest -Dtests.method=test -Dtests.seed=ECCB97A5D970679F -Dtests.slow=true -Dtests.locale=ko-KR -Dtests.timezone=Asia/Seoul -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 215s J1 | PeerSyncReplicationTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: timeout waiting to see all nodes active
[junit4] > at __randomizedtesting.SeedInfo.seed([ECCB97A5D970679F:649FA87F778C0A67]:0)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:326)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:277)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:259)
[junit4] > at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:138)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2902995 INFO (SUITE-PeerSyncReplicationTest-seed#[ECCB97A5D970679F]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_ECCB97A5D970679F-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {other_tl1=PostingsFormat(name=Memory), range_facet_l_dv=PostingsFormat(name=LuceneVarGapFixedInterval), rnd_s=PostingsFormat(name=Memory), multiDefault=PostingsFormat(name=Memory), a_t=PostingsFormat(name=Memory), intDefault=PostingsFormat(name=Memory), a_i1=PostingsFormat(name=Memory), id=PostingsFormat(name=LuceneVarGapFixedInterval), range_facet_i_dv=PostingsFormat(name=Memory), text=FSTOrd50, range_facet_l=PostingsFormat(name=Memory), timestamp=PostingsFormat(name=Memory)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Memory), _version_=DocValuesFormat(name=Lucene70), range_facet_i_dv=DocValuesFormat(name=Lucene70), intDvoDefault=DocValuesFormat(name=Lucene70), timestamp=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=987, maxMBSortInHeap=5.369901637346038, sim=RandomSimilarity(queryNorm=true): {}, locale=ko-KR, timezone=Asia/Seoul
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=113352272,total=363687936
[junit4] 2> NOTE: All tests run in this JVM: [TestNRTOpen, OverseerTaskQueueTest, TestLMDirichletSimilarityFactory, TestSolrQueryParserDefaultOperatorResource, NotRequiredUniqueKeyTest, UpdateParamsTest, TestExportWriter, ShufflingReplicaListTransformerTest, TestRangeQuery, OutOfBoxZkACLAndCredentialsProvidersTest, DirectUpdateHandlerOptimizeTest, SpellCheckCollatorTest, SSLMigrationTest, TestUseDocValuesAsStored2, TestSolrQueryParser, BitVectorTest, DeleteInactiveReplicaTest, TestFoldingMultitermQuery, CollectionsAPISolrJTest, TestWriterPerf, VersionInfoTest, TestRestManager, TestCoreDiscovery, InfoHandlerTest, RequestLoggingTest, TestShardHandlerFactory, TestDocTermOrds, SolrXmlInZkTest, TestIndexSearcher, TestCollectionAPI, TestSQLHandlerNonCloud, TestStressRecovery, JSONWriterTest, DistribCursorPagingTest, TestSolrDeletionPolicy2, CdcrUpdateLogTest, LeaderInitiatedRecoveryOnShardRestartTest, TestManagedSynonymFilterFactory, TestEmbeddedSolrServerConstructors, SuggesterTSTTest, ShardRoutingCustomTest, LoggingHandlerTest, TestCorePropertiesReload, TestSegmentSorting, TestScoreJoinQPNoScore, CreateCollectionCleanupTest, MinimalSchemaTest, DateMathParserTest, TestCloudPivotFacet, TestDistributedStatsComponentCardinality, TestSimpleTrackingShardHandler, ResponseLogComponentTest, TestDynamicFieldResource, TestLegacyFieldCache, TestQuerySenderListener, CoreSorterTest, BadIndexSchemaTest, PreAnalyzedUpdateProcessorTest, TestComponentsName, TestAnalyzedSuggestions, TestPointFields, TestJsonRequest, TestFieldCacheSort, AtomicUpdatesTest, DistributedFacetPivotLargeTest, OverseerCollectionConfigSetProcessorTest, DistributedIntervalFacetingTest, FieldAnalysisRequestHandlerTest, TestRTimerTree, DistributedVersionInfoTest, DeleteShardTest, TestDFISimilarityFactory, AutoCommitTest, TestLRUStatsCache, MissingSegmentRecoveryTest, BasicZkTest, TestZkChroot, DistributedTermsComponentTest, TestFileDictionaryLookup, TestManagedResource, TestSearchPerf, WrapperMergePolicyFactoryTest, SolrCoreMetricManagerTest, TestGraphMLResponseWriter, CircularListTest, SortByFunctionTest, TestOverriddenPrefixQueryForCustomFieldType, ScriptEngineTest, TestSolrConfigHandlerConcurrent, EnumFieldTest, TestCloudRecovery, LeaderFailureAfterFreshStartTest, TestRawTransformer, PeerSyncWithIndexFingerprintCachingTest, TestAuthorizationFramework, TestSchemalessBufferedUpdates, BlockJoinFacetDistribTest, DateRangeFieldTest, TestSubQueryTransformerCrossCore, SimpleMLTQParserTest, TestDefaultStatsCache, TestHighFrequencyDictionaryFactory, TestFieldCacheVsDocValues, SolrCLIZkUtilsTest, SignatureUpdateProcessorFactoryTest, DistributedFacetPivotSmallAdvancedTest, ConvertedLegacyTest, SpellingQueryConverterTest, OpenCloseCoreStressTest, HLLUtilTest, ReturnFieldsTest, UpdateRequestProcessorFactoryTest, SpellCheckComponentTest, TestSolrCloudWithHadoopAuthPlugin, TestStressLiveNodes, TriLevelCompositeIdRoutingTest, SolrMetricReporterTest, BooleanFieldTest, CacheHeaderTest, SolrGangliaReporterTest, RulesTest, TestUniqueKeyFieldResource, QueryResultKeyTest, TestFieldResource, AliasIntegrationTest, TestSolrCloudWithKerberosAlt, TestDistribDocBasedVersion, TestCustomSort, TestLMJelinekMercerSimilarityFactory, DistribJoinFromCollectionTest, CdcrReplicationHandlerTest, TestReplicaProperties, TestRandomCollapseQParserPlugin, DefaultValueUpdateProcessorTest, AnalyticsQueryTest, TestReversedWildcardFilterFactory, URLClassifyProcessorTest, TestFieldTypeResource, SolrCmdDistributorTest, TestSolrCoreProperties, PreAnalyzedFieldManagedSchemaCloudTest, BlockDirectoryTest, ShardSplitTest, NoCacheHeaderTest, CollectionReloadTest, ZkSolrClientTest, HdfsRecoverLeaseTest, ConfigSetsAPITest, CleanupOldIndexTest, CurrencyFieldOpenExchangeTest, TestDelegationWithHadoopAuth, StatsComponentTest, SolrIndexSplitterTest, TestSystemIdResolver, TestDocBasedVersionConstraints, IndexSchemaTest, HighlighterConfigTest, TestExactSharedStatsCache, DebugComponentTest, TestUtils, TestUnifiedSolrHighlighter, TestSortByMinMaxFunction, DistributedExpandComponentTest, HdfsRestartWhileUpdatingTest, JavabinLoaderTest, ShardRoutingTest, TestRandomFaceting, TestRandomDVFaceting, TestDistributedSearch, DistributedSpellCheckComponentTest, TestRealTimeGet, TestJoin, HardAutoCommitTest, QueryElevationComponentTest, TestSort, TestFiltering, TestFunctionQuery, TestLazyCores, SoftAutoCommitTest, CurrencyFieldXmlFileTest, SimplePostToolTest, TestExtendedDismaxParser, SolrRequestParserTest, SuggesterTest, TestStressLucene, TestTrie, SpatialFilterTest, PolyFieldTest, WordBreakSolrSpellCheckerTest, SolrCoreCheckLockOnStartupTest, TestAtomicUpdateErrorCases, TestRemoteStreaming, SolrInfoMBeanTest, LukeRequestHandlerTest, FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest, RequestHandlersTest, TestValueSourceCache, TestIndexingPerformance, TestCollationField, TestCSVResponseWriter, UniqFieldsUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, CSVRequestHandlerTest, AlternateDirectoryTest, TestSolrIndexConfig, CopyFieldTest, SolrIndexConfigTest, BadComponentTest, TestMergePolicyConfig, TestDocSet, NumericFieldsTest, TestElisionMultitermQuery, TestConfig, OutputWriterTest, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, TestPostingsSolrHighlighter, TestLuceneMatchVersion, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestCodecSupport, SynonymTokenizerTest, EchoParamsTest, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestIBSimilarityFactory, TestBM25SimilarityFactory, TestJmxMonitoredMap, TestFastLRUCache, TestSuggestSpellingConverter, RAMDirectoryFactoryTest, SliceStateTest, SystemInfoHandlerTest, UUIDFieldTest, CursorPagingTest, CdcrRequestHandlerTest, CdcrVersionReplicationTest, CollectionStateFormat2Test, CollectionsAPIAsyncDistributedZkTest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DeleteStatusTest, DistributedQueueTest, ForceLeaderTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, OverriddenZkACLAndCredentialsProvidersTest, PeerSyncReplicationTest]
[junit4] Completed [518/684 (1!)] on J1 in 215.22s, 1 test, 1 failure <<< FAILURES!
[...truncated 62983 lines...]
[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 3798
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/3798/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC
1 tests failed.
FAILED: org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingSorting
Error Message:
Should have exactly 4 documents returned expected:<4> but was:<3>
Stack Trace:
java.lang.AssertionError: Should have exactly 4 documents returned expected:<4> but was:<3>
at __randomizedtesting.SeedInfo.seed([36B7C7B063946B0B:288FCFB81F3FD18B]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.apache.solr.cloud.DocValuesNotIndexedTest.checkSortOrder(DocValuesNotIndexedTest.java:259)
at org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingSorting(DocValuesNotIndexedTest.java:244)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12209 lines...]
[junit4] Suite: org.apache.solr.cloud.DocValuesNotIndexedTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/init-core-data-001
[junit4] 2> 2402936 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields
[junit4] 2> 2402943 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 2402945 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001
[junit4] 2> 2402945 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2402946 INFO (Thread-3490) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2402946 INFO (Thread-3490) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2403051 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:53916
[junit4] 2> 2403260 INFO (jetty-launcher-2861-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2403262 INFO (jetty-launcher-2861-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2403264 INFO (jetty-launcher-2861-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6417bf1d{/solr,null,AVAILABLE}
[junit4] 2> 2403264 INFO (jetty-launcher-2861-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5962d615{HTTP/1.1,[http/1.1]}{127.0.0.1:53918}
[junit4] 2> 2403264 INFO (jetty-launcher-2861-thread-1) [ ] o.e.j.s.Server Started @2433856ms
[junit4] 2> 2403264 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=53918}
[junit4] 2> 2403264 ERROR (jetty-launcher-2861-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2403265 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2403265 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2403265 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2403265 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T23:45:08.636Z
[junit4] 2> 2403269 INFO (jetty-launcher-2861-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@574b78af{/solr,null,AVAILABLE}
[junit4] 2> 2403269 INFO (jetty-launcher-2861-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7ab1375e{HTTP/1.1,[http/1.1]}{127.0.0.1:53919}
[junit4] 2> 2403270 INFO (jetty-launcher-2861-thread-2) [ ] o.e.j.s.Server Started @2433862ms
[junit4] 2> 2403270 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=53919}
[junit4] 2> 2403270 ERROR (jetty-launcher-2861-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2403271 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.0.0
[junit4] 2> 2403271 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2403271 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2403271 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-01-25T23:45:08.642Z
[junit4] 2> 2403280 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2403281 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2403293 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53916/solr
[junit4] 2> 2403297 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53916/solr
[junit4] 2> 2403367 INFO (jetty-launcher-2861-thread-2) [n:127.0.0.1:53919_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:53919_solr
[junit4] 2> 2403369 INFO (jetty-launcher-2861-thread-1) [n:127.0.0.1:53918_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53918_solr
[junit4] 2> 2403370 INFO (jetty-launcher-2861-thread-2) [n:127.0.0.1:53919_solr ] o.a.s.c.Overseer Overseer (id=97346381959790598-127.0.0.1:53919_solr-n_0000000000) starting
[junit4] 2> 2403373 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2403374 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2403393 INFO (jetty-launcher-2861-thread-2) [n:127.0.0.1:53919_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53919_solr
[junit4] 2> 2403396 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2403397 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2404081 INFO (jetty-launcher-2861-thread-1) [n:127.0.0.1:53918_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node1/.
[junit4] 2> 2404086 INFO (jetty-launcher-2861-thread-2) [n:127.0.0.1:53919_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/.
[junit4] 2> 2405587 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2405591 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53916/solr ready
[junit4] 2> 2405619 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&maxShardsPerNode=2&name=dv_coll&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2405623 INFO (OverseerThreadFactory-8238-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.CreateCollectionCmd Create collection dv_coll
[junit4] 2> 2405745 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard3_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard3&wt=javabin&version=2
[junit4] 2> 2405745 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard4_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard4&wt=javabin&version=2
[junit4] 2> 2405745 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard1_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard1&wt=javabin&version=2
[junit4] 2> 2405749 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard2_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard2&wt=javabin&version=2
[junit4] 2> 2405868 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2405868 INFO (zkCallback-2872-thread-3-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2406789 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2406790 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2406791 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2406796 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2406801 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.s.IndexSchema [dv_coll_shard2_replica1] Schema name=minimal
[junit4] 2> 2406806 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2406808 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.s.IndexSchema [dv_coll_shard1_replica1] Schema name=minimal
[junit4] 2> 2406811 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2406813 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard2_replica1' using configuration from collection dv_coll
[junit4] 2> 2406813 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.SolrCore [[dv_coll_shard2_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node1/dv_coll_shard2_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node1/./dv_coll_shard2_replica1/data/]
[junit4] 2> 2406815 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.s.IndexSchema [dv_coll_shard3_replica1] Schema name=minimal
[junit4] 2> 2406816 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.s.IndexSchema [dv_coll_shard4_replica1] Schema name=minimal
[junit4] 2> 2406817 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2406818 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2406819 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard1_replica1' using configuration from collection dv_coll
[junit4] 2> 2406819 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard3_replica1' using configuration from collection dv_coll
[junit4] 2> 2406819 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [[dv_coll_shard3_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/dv_coll_shard3_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/./dv_coll_shard3_replica1/data/]
[junit4] 2> 2406819 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.SolrCore [[dv_coll_shard1_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/dv_coll_shard1_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/./dv_coll_shard1_replica1/data/]
[junit4] 2> 2406821 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard4_replica1' using configuration from collection dv_coll
[junit4] 2> 2406821 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.SolrCore [[dv_coll_shard4_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node1/dv_coll_shard4_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node1/./dv_coll_shard4_replica1/data/]
[junit4] 2> 2406862 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2406862 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2406865 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2406865 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2406875 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6e42e2bb[dv_coll_shard2_replica1] main]
[junit4] 2> 2406876 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2406876 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2406879 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2406879 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2406880 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2406880 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2406881 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2406881 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2406883 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2406883 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2406883 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2406884 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2406884 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
[junit4] 2> 2406885 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2406885 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2406885 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 2406886 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2406886 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557542115482796032
[junit4] 2> 2406886 INFO (searcherExecutor-8243-thread-1-processing-n:127.0.0.1:53918_solr x:dv_coll_shard2_replica1 s:shard2 c:dv_coll) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.SolrCore [dv_coll_shard2_replica1] Registered new searcher Searcher@6e42e2bb[dv_coll_shard2_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2406887 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6517068e[dv_coll_shard3_replica1] main]
[junit4] 2> 2406890 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2406890 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@67d8b358[dv_coll_shard4_replica1] main]
[junit4] 2> 2406893 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2406894 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2406895 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2406895 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
[junit4] 2> 2406898 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 2406898 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2406898 INFO (searcherExecutor-8246-thread-1-processing-n:127.0.0.1:53918_solr x:dv_coll_shard4_replica1 s:shard4 c:dv_coll) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.SolrCore [dv_coll_shard4_replica1] Registered new searcher Searcher@67d8b358[dv_coll_shard4_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2406899 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557542115496427520
[junit4] 2> 2406903 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
[junit4] 2> 2406905 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 2406906 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2406906 INFO (searcherExecutor-8245-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard3_replica1 s:shard3 c:dv_coll) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] Registered new searcher Searcher@6517068e[dv_coll_shard3_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2406906 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557542115503767552
[junit4] 2> 2406913 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2406913 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2406913 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53918/solr/dv_coll_shard4_replica1/
[junit4] 2> 2406913 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2406913 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:53918/solr/dv_coll_shard4_replica1/ has no replicas
[junit4] 2> 2406917 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2406918 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2406918 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/
[junit4] 2> 2406918 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2406919 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/ has no replicas
[junit4] 2> 2406923 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2406923 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2406923 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/
[junit4] 2> 2406923 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2406923 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/ has no replicas
[junit4] 2> 2406924 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53918/solr/dv_coll_shard4_replica1/ shard4
[junit4] 2> 2406930 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/ shard2
[junit4] 2> 2406930 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/ shard3
[junit4] 2> 2406936 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5c9c30de[dv_coll_shard1_replica1] main]
[junit4] 2> 2406939 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2406940 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2406940 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
[junit4] 2> 2406942 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 2406942 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2406942 INFO (searcherExecutor-8244-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard1_replica1 s:shard1 c:dv_coll) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.SolrCore [dv_coll_shard1_replica1] Registered new searcher Searcher@5c9c30de[dv_coll_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2406942 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557542115541516288
[junit4] 2> 2406955 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2406955 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2406955 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/
[junit4] 2> 2406955 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2406955 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/ has no replicas
[junit4] 2> 2406963 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/ shard1
[junit4] 2> 2407075 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2407076 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2407081 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2407087 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 x:dv_coll_shard4_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard4_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard4&wt=javabin&version=2} status=0 QTime=1342
[junit4] 2> 2407097 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2407098 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2407108 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2407103 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 x:dv_coll_shard2_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard2_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard2&wt=javabin&version=2} status=0 QTime=1355
[junit4] 2> 2407115 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 x:dv_coll_shard3_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard3_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard3&wt=javabin&version=2} status=0 QTime=1370
[junit4] 2> 2407129 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 x:dv_coll_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=dv_coll_shard1_replica1&action=CREATE&numShards=4&collection=dv_coll&shard=shard1&wt=javabin&version=2} status=0 QTime=1383
[junit4] 2> 2407138 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2407229 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2407229 INFO (zkCallback-2872-thread-3-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2407638 INFO (OverseerCollectionConfigSetProcessor-97346381959790598-127.0.0.1:53919_solr-n_0000000000) [n:127.0.0.1:53919_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2408140 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&maxShardsPerNode=2&name=dv_coll&action=CREATE&numShards=4&wt=javabin&version=2} status=0 QTime=2521
[junit4] 2> 2408189 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SchemaManager managed schema loaded . version : 0
[junit4] 2> 2408199 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.IndexSchema [dv_coll_shard3_replica1] Schema name=minimal
[junit4] 2> 2408201 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2408214 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.ZkController Persisted config data to node /configs/conf1/managed-schema
[junit4] 2> 2408215 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
[junit4] 2> 2408215 INFO (zkCallback-2871-thread-2-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
[junit4] 2> 2408215 INFO (zkCallback-2872-thread-3-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
[junit4] 2> 2408218 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/conf1/managed-schema, has occurred - updating schema from ZooKeeper ...
[junit4] 2> 2408218 INFO (zkCallback-2871-thread-2-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
[junit4] 2> 2408220 INFO (zkCallback-2871-thread-2-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.IndexSchema [dv_coll_shard2_replica1] Schema name=minimal
[junit4] 2> 2408223 INFO (zkCallback-2871-thread-2-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2408224 INFO (zkCallback-2871-thread-2-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 5 ms
[junit4] 2> 2408224 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
[junit4] 2> 2408226 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.IndexSchema [dv_coll_shard1_replica1] Schema name=minimal
[junit4] 2> 2408229 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
[junit4] 2> 2408229 INFO (Thread-3509) [n:127.0.0.1:53919_solr ] o.a.s.c.SolrCore config update listener called for core dv_coll_shard3_replica1
[junit4] 2> 2408230 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2408231 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 6 ms
[junit4] 2> 2408231 INFO (Thread-3510) [n:127.0.0.1:53918_solr ] o.a.s.c.SolrCore config update listener called for core dv_coll_shard2_replica1
[junit4] 2> 2408232 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.IndexSchema [dv_coll_shard4_replica1] Schema name=minimal
[junit4] 2> 2408236 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2408236 INFO (zkCallback-2871-thread-1-processing-n:127.0.0.1:53918_solr) [n:127.0.0.1:53918_solr ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 6 ms
[junit4] 2> 2408237 INFO (Thread-3509) [n:127.0.0.1:53919_solr ] o.a.s.c.SolrCore core reload dv_coll_shard3_replica1
[junit4] 2> 2408238 INFO (Thread-3510) [n:127.0.0.1:53918_solr ] o.a.s.c.SolrCore config update listener called for core dv_coll_shard4_replica1
[junit4] 2> 2408243 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2408250 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.IndexSchema [dv_coll_shard3_replica1] Schema name=minimal
[junit4] 2> 2408254 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2408255 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.CoreContainer Reloading SolrCore 'dv_coll_shard3_replica1' using configuration from collection dv_coll
[junit4] 2> 2408256 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [[dv_coll_shard3_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/dv_coll_shard3_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/./dv_coll_shard3_replica1/data/]
[junit4] 2> 2408278 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2408278 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2408278 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1f08b308[dv_coll_shard3_replica1] main]
[junit4] 2> 2408280 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2408281 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2408281 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
[junit4] 2> 2408282 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 1 is already the latest
[junit4] 2> 2408282 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2408282 INFO (searcherExecutor-8263-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard3_replica1 s:shard3 c:dv_coll r:core_node2) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] Registered new searcher Searcher@1f08b308[dv_coll_shard3_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408283 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1557542116947656704
[junit4] 2> 2408291 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 2408291 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7765a366[dv_coll_shard3_replica1] main]
[junit4] 2> 2408292 INFO (searcherExecutor-8263-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard3_replica1 s:shard3 c:dv_coll r:core_node2) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] Registered new searcher Searcher@7765a366[dv_coll_shard3_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408292 INFO (zkCallback-2872-thread-3-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from ZooKeeper
[junit4] 2> 2408296 INFO (zkCallback-2872-thread-3-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.IndexSchema [dv_coll_shard3_replica1] Schema name=minimal
[junit4] 2> 2408323 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.ManagedIndexSchema Waiting up to 599 secs for 3 replicas to apply schema update version 1 for collection dv_coll
[junit4] 2> 2408323 INFO (Thread-3509) [n:127.0.0.1:53919_solr ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 2408323 INFO (zkCallback-2872-thread-3-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2408323 INFO (zkCallback-2872-thread-3-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 30 ms
[junit4] 2> 2408329 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.c.S.Request [dv_coll_shard1_replica1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=1&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2408330 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.c.S.Request [dv_coll_shard2_replica1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=1&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2408331 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.c.S.Request [dv_coll_shard4_replica1] webapp=/solr path=/schema/zkversion params={refreshIfBelowVersion=1&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2408332 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.ManagedIndexSchema Took 40.0ms for 3 replicas to apply schema update version 1 for collection dv_coll
[junit4] 2> 2408333 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.S.Request [dv_coll_shard3_replica1] webapp=/solr path=/schema params={wt=javabin&version=2} status=0 QTime=147
[junit4] 2> 2408333 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@27b1385a
[junit4] 2> 2408333 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.dv_coll.shard3.replica1
[junit4] 2> 2408334 INFO (Thread-3509) [n:127.0.0.1:53919_solr ] o.a.s.s.IndexSchema [dv_coll_shard3_replica1] Schema name=minimal
[junit4] 2> 2408348 INFO (Thread-3509) [n:127.0.0.1:53919_solr ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2408352 INFO (Thread-3509) [n:127.0.0.1:53919_solr ] o.a.s.c.CoreContainer Reloading SolrCore 'dv_coll_shard3_replica1' using configuration from collection dv_coll
[junit4] 2> 2408357 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [[dv_coll_shard3_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/dv_coll_shard3_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001/tempDir-001/node2/./dv_coll_shard3_replica1/data/]
[junit4] 2> 2408366 INFO (TEST-DocValuesNotIndexedTest.testGroupingSorting-seed#[36B7C7B063946B0B]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testGroupingSorting
[junit4] 2> 2408375 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557542117040979968)} 0 3
[junit4] 2> 2408377 INFO (qtp1113000480-16297) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica1] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557542117039931392)} 0 5
[junit4] 2> 2408377 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557542117039931392)} 0 6
[junit4] 2> 2408378 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557542117037834240)} 0 10
[junit4] 2> 2408379 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2408379 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2408379 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@453142b6[dv_coll_shard3_replica1] main]
[junit4] 2> 2408380 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408380 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2408381 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408381 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1d70f30[dv_coll_shard1_replica1] main]
[junit4] 2> 2408382 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2408382 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408382 INFO (searcherExecutor-8244-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard1_replica1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.c.SolrCore [dv_coll_shard1_replica1] Registered new searcher Searcher@1d70f30[dv_coll_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408382 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6e7cced2[dv_coll_shard2_replica1] main]
[junit4] 2> 2408382 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 2408382 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408382 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2408382 INFO (searcherExecutor-8243-thread-1-processing-n:127.0.0.1:53918_solr x:dv_coll_shard2_replica1 s:shard2 c:dv_coll r:core_node4) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.c.SolrCore [dv_coll_shard2_replica1] Registered new searcher Searcher@6e7cced2[dv_coll_shard2_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408382 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408383 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 2408383 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408383 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2408384 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6aad5317[dv_coll_shard4_replica1] main]
[junit4] 2> 2408384 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 2408384 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408384 INFO (searcherExecutor-8246-thread-1-processing-n:127.0.0.1:53918_solr x:dv_coll_shard4_replica1 s:shard4 c:dv_coll r:core_node1) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.c.SolrCore [dv_coll_shard4_replica1] Registered new searcher Searcher@6aad5317[dv_coll_shard4_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408384 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 1
[junit4] 2> 2408385 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 2408385 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
[junit4] 2> 2408386 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.ZkIndexSchemaReader Current schema version 1 is already the latest
[junit4] 2> 2408386 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2408386 INFO (searcherExecutor-8268-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard3_replica1 s:shard3 c:dv_coll r:core_node2) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] Registered new searcher Searcher@453142b6[dv_coll_shard3_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408386 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408387 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 4
[junit4] 2> 2408387 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 8
[junit4] 2> 2408393 INFO (qtp1113000480-16299) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&wt=javabin&version=2}{add=[4 (1557542117058805760)]} 0 4
[junit4] 2> 2408395 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&wt=javabin&version=2}{add=[1 (1557542117059854336)]} 0 4
[junit4] 2> 2408451 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 2408453 INFO (qtp1276199975-16301) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&wt=javabin&version=2}{add=[2 (1557542117059854336), 3 (1557542117063000064)]} 0 64
[junit4] 2> 2408455 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408455 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408455 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2f684653
[junit4] 2> 2408455 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408455 INFO (qtp1113000480-16295) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408455 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@4b0c9b2e
[junit4] 2> 2408455 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@204e25a6
[junit4] 2> 2408455 INFO (qtp1113000480-16295) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2408459 INFO (qtp1113000480-16295) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408459 INFO (qtp1113000480-16295) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 4
[junit4] 2> 2408466 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5451e0e3[dv_coll_shard3_replica1] main]
[junit4] 2> 2408467 INFO (Thread-3509) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore config update listener called for core dv_coll_shard1_replica1
[junit4] 2> 2408467 INFO (searcherExecutor-8268-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard3_replica1 s:shard3 c:dv_coll r:core_node2) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] Registered new searcher Searcher@5451e0e3[dv_coll_shard3_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C1)))}
[junit4] 2> 2408498 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@43c56d3a[dv_coll_shard2_replica1] main]
[junit4] 2> 2408499 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408499 INFO (searcherExecutor-8243-thread-1-processing-n:127.0.0.1:53918_solr x:dv_coll_shard2_replica1 s:shard2 c:dv_coll r:core_node4) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.c.SolrCore [dv_coll_shard2_replica1] Registered new searcher Searcher@43c56d3a[dv_coll_shard2_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C1)))}
[junit4] 2> 2408499 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 44
[junit4] 2> 2408515 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4c4bd8da[dv_coll_shard1_replica1] main]
[junit4] 2> 2408515 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408515 INFO (searcherExecutor-8244-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard1_replica1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.c.SolrCore [dv_coll_shard1_replica1] Registered new searcher Searcher@4c4bd8da[dv_coll_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C1)))}
[junit4] 2> 2408516 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 61
[junit4] 2> 2408517 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3f2ccaac[dv_coll_shard3_replica1] main]
[junit4] 2> 2408517 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408517 INFO (searcherExecutor-8263-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard3_replica1 s:shard3 c:dv_coll r:core_node2) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] Registered new searcher Searcher@3f2ccaac[dv_coll_shard3_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C1) Uninverting(_1(7.0.0):C1)))}
[junit4] 2> 2408517 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 62
[junit4] 2> 2408517 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@5f5b6715
[junit4] 2> 2408518 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.dv_coll.shard3.replica1
[junit4] 2> 2408518 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&commit=true&wt=javabin&version=2}{commit=} 0 64
[junit4] 2> 2408522 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.c.S.Request [dv_coll_shard2_replica1] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=dv_coll:3&fl=id&shards.purpose=4&start=0&fsv=true&sort=intGSF+asc,id+asc&shard.url=http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/&rows=100&version=2&q=*:*&NOW=1485387913890&isShard=true&wt=javabin} hits=1 status=0 QTime=0
[junit4] 2> 2408523 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.c.S.Request [dv_coll_shard4_replica1] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=dv_coll:3&fl=id&shards.purpose=4&start=0&fsv=true&sort=intGSF+asc,id+asc&shard.url=http://127.0.0.1:53918/solr/dv_coll_shard4_replica1/&rows=100&version=2&q=*:*&NOW=1485387913890&isShard=true&wt=javabin} hits=0 status=0 QTime=0
[junit4] 2> 2408523 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.S.Request [dv_coll_shard3_replica1] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=dv_coll:3&fl=id&shards.purpose=4&start=0&fsv=true&sort=intGSF+asc,id+asc&shard.url=http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/&rows=100&version=2&q=*:*&NOW=1485387913890&isShard=true&wt=javabin} hits=1 status=0 QTime=0
[junit4] 2> 2408524 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.c.S.Request [dv_coll_shard1_replica1] webapp=/solr path=/select params={df=text&distrib=false&_stateVer_=dv_coll:3&fl=id&shards.purpose=4&start=0&fsv=true&sort=intGSF+asc,id+asc&shard.url=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&rows=100&version=2&q=*:*&NOW=1485387913890&isShard=true&wt=javabin} hits=1 status=0 QTime=0
[junit4] 2> 2408526 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.c.S.Request [dv_coll_shard2_replica1] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&_stateVer_=dv_coll:3&shards.purpose=64&NOW=1485387913890&ids=4&isShard=true&shard.url=http://127.0.0.1:53918/solr/dv_coll_shard2_replica1/&rows=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2408526 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.c.S.Request [dv_coll_shard1_replica1] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&_stateVer_=dv_coll:3&shards.purpose=64&NOW=1485387913890&ids=1&isShard=true&shard.url=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&rows=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2408526 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.S.Request [dv_coll_shard3_replica1] webapp=/solr path=/select params={q=*:*&df=text&distrib=false&_stateVer_=dv_coll:3&shards.purpose=64&NOW=1485387913890&ids=2&isShard=true&shard.url=http://127.0.0.1:53919/solr/dv_coll_shard3_replica1/&rows=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2408527 INFO (qtp1113000480-16297) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.c.S.Request [dv_coll_shard4_replica1] webapp=/solr path=/select params={q=*:*&_stateVer_=dv_coll:3&sort=intGSF+asc,id+asc&rows=100&wt=javabin&version=2} hits=3 status=0 QTime=7
[junit4] 2> 2408528 INFO (TEST-DocValuesNotIndexedTest.testGroupingSorting-seed#[36B7C7B063946B0B]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testGroupingSorting
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=DocValuesNotIndexedTest -Dtests.method=testGroupingSorting -Dtests.seed=36B7C7B063946B0B -Dtests.slow=true -Dtests.locale=en-US -Dtests.timezone=Pacific/Enderbury -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] FAILURE 0.20s J1 | DocValuesNotIndexedTest.testGroupingSorting <<<
[junit4] > Throwable #1: java.lang.AssertionError: Should have exactly 4 documents returned expected:<4> but was:<3>
[junit4] > at __randomizedtesting.SeedInfo.seed([36B7C7B063946B0B:288FCFB81F3FD18B]:0)
[junit4] > at org.apache.solr.cloud.DocValuesNotIndexedTest.checkSortOrder(DocValuesNotIndexedTest.java:259)
[junit4] > at org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingSorting(DocValuesNotIndexedTest.java:244)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2408558 INFO (TEST-DocValuesNotIndexedTest.testGroupingDocAbsent-seed#[36B7C7B063946B0B]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testGroupingDocAbsent
[junit4] 2> 2408562 INFO (qtp1113000480-16295) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3f5d8bd1[dv_coll_shard2_replica1] realtime]
[junit4] 2> 2408562 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1c36232c[dv_coll_shard1_replica1] realtime]
[junit4] 2> 2408563 INFO (qtp1113000480-16295) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica1] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:53918/solr/dv_coll_shard4_replica1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557542117237063680)} 0 3
[junit4] 2> 2408563 INFO (qtp1276199975-16301) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@454f76a7[dv_coll_shard3_replica1] realtime]
[junit4] 2> 2408563 INFO (qtp1276199975-16303) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:53918/solr/dv_coll_shard4_replica1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557542117237063680)} 0 3
[junit4] 2> 2408563 INFO (qtp1276199975-16301) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={update.distrib=TOLEADER&distrib.from=http://127.0.0.1:53918/solr/dv_coll_shard4_replica1/&wt=javabin&version=2}{deleteByQuery=*:* (-1557542117237063680)} 0 3
[junit4] 2> 2408563 INFO (qtp1113000480-16299) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1557542117237063680)} 0 5
[junit4] 2> 2408566 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408566 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408566 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408566 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2f684653
[junit4] 2> 2408566 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@204e25a6
[junit4] 2> 2408566 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@4b0c9b2e
[junit4] 2> 2408566 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408566 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2408566 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@14cba9f9[dv_coll_shard2_replica1] main]
[junit4] 2> 2408567 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408567 INFO (qtp1113000480-16294) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard4_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 0
[junit4] 2> 2408567 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408567 INFO (searcherExecutor-8243-thread-1-processing-n:127.0.0.1:53918_solr x:dv_coll_shard2_replica1 s:shard2 c:dv_coll r:core_node4) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.c.SolrCore [dv_coll_shard2_replica1] Registered new searcher Searcher@14cba9f9[dv_coll_shard2_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408567 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@53bc3164[dv_coll_shard1_replica1] main]
[junit4] 2> 2408568 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408568 INFO (searcherExecutor-8244-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard1_replica1 s:shard1 c:dv_coll r:core_node3) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.c.SolrCore [dv_coll_shard1_replica1] Registered new searcher Searcher@53bc3164[dv_coll_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408568 INFO (qtp1113000480-16292) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 2408568 INFO (qtp1276199975-16307) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 3
[junit4] 2> 2408569 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@21174ffa[dv_coll_shard3_replica1] main]
[junit4] 2> 2408569 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408569 INFO (searcherExecutor-8268-thread-1-processing-n:127.0.0.1:53919_solr x:dv_coll_shard3_replica1 s:shard3 c:dv_coll r:core_node2) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.c.SolrCore [dv_coll_shard3_replica1] Registered new searcher Searcher@21174ffa[dv_coll_shard3_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2408569 INFO (qtp1276199975-16305) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:53919/solr/dv_coll_shard1_replica1/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 3
[junit4] 2> 2408570 INFO (qtp1276199975-16308) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 4
[junit4] 2> 2408573 INFO (qtp1276199975-16304) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard1_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&wt=javabin&version=2}{add=[1 (1557542117249646592)]} 0 2
[junit4] 2> 2408574 INFO (qtp1276199975-16306) [n:127.0.0.1:53919_solr c:dv_coll s:shard3 r:core_node2 x:dv_coll_shard3_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard3_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&wt=javabin&version=2}{add=[2 (1557542117250695168), 3 (1557542117252792320)]} 0 2
[junit4] 2> 2408574 INFO (qtp1113000480-16296) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.p.LogUpdateProcessorFactory [dv_coll_shard2_replica1] webapp=/solr path=/update params={_stateVer_=dv_coll:3&wt=javabin&version=2}{add=[4 (1557542117249646592)]} 0 3
[junit4] 2> 2408576 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408576 INFO (qtp1276199975-16301) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408577 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 2408577 INFO (qtp1276199975-16301) [n:127.0.0.1:53919_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@204e25a6
[junit4] 2> 2408577 INFO (qtp1113000480-16297) [n:127.0.0.1:53918_solr c:dv_coll s:shard2 r:core_node4 x:dv_coll_shard2_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2408577 INFO (qtp1113000480-16298) [n:127.0.0.1:53918_solr c:dv_coll s:shard4 r:core_node1 x:dv_coll_shard4_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2408577 INFO (q
[...truncated too long message...]
e_nodes]
[junit4] 2> 2410675 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.c.Overseer Overseer (id=97346381959790598-127.0.0.1:53919_solr-n_0000000000) closing
[junit4] 2> 2410675 INFO (jetty-launcher-2861-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 2410675 INFO (OverseerStateUpdate-97346381959790598-127.0.0.1:53919_solr-n_0000000000) [n:127.0.0.1:53919_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:53919_solr
[junit4] 2> 2410675 INFO (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 2410676 INFO (jetty-launcher-2861-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6417bf1d{/solr,null,UNAVAILABLE}
[junit4] 2> 2412178 WARN (zkCallback-2872-thread-1-processing-n:127.0.0.1:53919_solr) [n:127.0.0.1:53919_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 2412179 INFO (jetty-launcher-2861-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
[junit4] 2> 2412179 INFO (jetty-launcher-2861-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@574b78af{/solr,null,UNAVAILABLE}
[junit4] 2> 2412201 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53916 53916
[junit4] 2> 2416494 INFO (Thread-3490) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53916 53916
[junit4] 2> 2419026 WARN (Thread-3490) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/configs/conf1/managed-schema
[junit4] 2> 4 /solr/configs/conf1
[junit4] 2> 3 /solr/aliases.json
[junit4] 2> 3 /solr/clusterprops.json
[junit4] 2> 2 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/clusterstate.json
[junit4] 2> 2 /solr/configs/conf1/managed-schema
[junit4] 2> 2 /solr/collections/dv_coll/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 13 /solr/overseer/queue
[junit4] 2> 7 /solr/overseer/collection-queue-work
[junit4] 2> 3 /solr/live_nodes
[junit4] 2> 3 /solr/collections
[junit4] 2> 3 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> 2419027 INFO (SUITE-DocValuesNotIndexedTest-seed#[36B7C7B063946B0B]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DocValuesNotIndexedTest_36B7C7B063946B0B-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {_version_=FSTOrd50, id=PostingsFormat(name=LuceneFixedGap)}, docValues:{floatField=DocValuesFormat(name=Lucene70), doubleField=DocValuesFormat(name=Asserting), intFieldMulti=DocValuesFormat(name=Lucene70), boolFieldMulti=DocValuesFormat(name=Asserting), floatGSL=DocValuesFormat(name=Lucene70), longFieldMulti=DocValuesFormat(name=Memory), intField=DocValuesFormat(name=Asserting), boolField=DocValuesFormat(name=Lucene70), stringFieldMulti=DocValuesFormat(name=Lucene70), dateFieldMulti=DocValuesFormat(name=Memory), longGSF=DocValuesFormat(name=Lucene70), doubleFieldMulti=DocValuesFormat(name=Lucene70), stringGSL=DocValuesFormat(name=Memory), doubleGSF=DocValuesFormat(name=Asserting), doubleGSL=DocValuesFormat(name=Memory), stringGSF=DocValuesFormat(name=Asserting), intGSL=DocValuesFormat(name=Asserting), longField=DocValuesFormat(name=Lucene70), dateField=DocValuesFormat(name=Lucene70), dateGSF=DocValuesFormat(name=Lucene70), boolGSF=DocValuesFormat(name=Lucene70), longGSL=DocValuesFormat(name=Lucene70), boolGSL=DocValuesFormat(name=Lucene70), floatFieldMulti=DocValuesFormat(name=Asserting), floatGSF=DocValuesFormat(name=Lucene70), dateGSL=DocValuesFormat(name=Lucene70), intGSF=DocValuesFormat(name=Memory), stringField=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=994, maxMBSortInHeap=5.7251959539859705, sim=RandomSimilarity(queryNorm=true): {}, locale=en-US, timezone=Pacific/Enderbury
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_121 (64-bit)/cpus=3,threads=1,free=244125368,total=490733568
[junit4] 2> NOTE: All tests run in this JVM: [TestSolrCloudWithKerberosAlt, DocValuesMissingTest, SolrIndexMetricsTest, TestComplexPhraseQParserPlugin, VMParamsZkACLAndCredentialsProvidersTest, SolrCLIZkUtilsTest, DistanceUnitsTest, TestScoreJoinQPScore, TestSchemaSimilarityResource, TestBadConfig, SolrMetricsIntegrationTest, TestCloudSchemaless, DeleteInactiveReplicaTest, TestImplicitCoreProperties, TestBulkSchemaAPI, TestTrackingShardHandlerFactory, UpdateParamsTest, AsyncCallRequestStatusResponseTest, SuggesterTest, TestDynamicFieldCollectionResource, TestInitParams, BadIndexSchemaTest, BasicFunctionalityTest, TestConfigSetImmutable, LukeRequestHandlerTest, SolrMetricManagerTest, TestDFRSimilarityFactory, ShardSplitTest, HdfsBasicDistributedZkTest, TestJoin, TestHdfsBackupRestoreCore, EchoParamsTest, TestXmlQParserPlugin, StatsComponentTest, TestInitQParser, TestManagedSchemaThreadSafety, OverseerRolesTest, TestSchemaNameResource, DistributedFacetExistsSmallTest, DistributedIntervalFacetingTest, TestQueryUtils, URLClassifyProcessorTest, TestSegmentSorting, RankQueryTest, AnalyticsMergeStrategyTest, TestSolrConfigHandler, TestCrossCoreJoin, CdcrBootstrapTest, CollectionsAPIDistributedZkTest, TestFuzzyAnalyzedSuggestions, RegexBoostProcessorTest, DataDrivenBlockJoinTest, CdcrVersionReplicationTest, ReturnFieldsTest, TestConfig, TestClassicSimilarityFactory, TestLMJelinekMercerSimilarityFactory, FullHLLTest, UUIDFieldTest, HdfsSyncSliceTest, DistributedTermsComponentTest, TestTolerantUpdateProcessorCloud, HdfsNNFailoverTest, ForceLeaderTest, TestComponentsName, TestJmxMonitoredMap, TestSolrXml, MissingSegmentRecoveryTest, CoreAdminCreateDiscoverTest, AtomicUpdatesTest, TestLMDirichletSimilarityFactory, SpellCheckComponentTest, TestManagedStopFilterFactory, TestDistributedMissingSort, LeaderInitiatedRecoveryOnShardRestartTest, TestBlendedInfixSuggestions, RequestHandlersTest, DefaultValueUpdateProcessorTest, DistributedSuggestComponentTest, TestRangeQuery, TestDistribIDF, JsonLoaderTest, TestReversedWildcardFilterFactory, TestExpandComponent, SignatureUpdateProcessorFactoryTest, TemplateUpdateProcessorTest, DeleteNodeTest, HighlighterConfigTest, TestFreeTextSuggestions, TestRTimerTree, TestStandardQParsers, BlockCacheTest, CollectionTooManyReplicasTest, SpatialRPTFieldTypeTest, AnalysisErrorHandlingTest, TestBackupRepositoryFactory, TestZkChroot, HdfsTlogReplayBufferedWhileIndexingTest, ReplaceNodeTest, ResponseLogComponentTest, ShowFileRequestHandlerTest, TestSolrCoreParser, TestSolrQueryParserResource, TestSearchPerf, HLLUtilTest, RestartWhileUpdatingTest, TestSQLHandler, TestLuceneMatchVersion, QueryParsingTest, BasicAuthIntegrationTest, TermVectorComponentTest, TestCloudDeleteByQuery, DistanceFunctionTest, MetricUtilsTest, TestSolrCoreProperties, TestFoldingMultitermQuery, TestCopyFieldCollectionResource, TestExceedMaxTermLength, TestTrieFacet, TestSolrDeletionPolicy2, TestBlobHandler, TestCloudManagedSchema, TestTrie, TestSolr4Spatial, TestSubQueryTransformerDistrib, SolrIndexSplitterTest, TestSmileRequest, CdcrReplicationHandlerTest, PreAnalyzedFieldTest, SolrInfoMBeanTest, TestOnReconnectListenerSupport, TestEmbeddedSolrServerConstructors, SolrCoreCheckLockOnStartupTest, DeleteShardTest, DistributedFacetPivotSmallTest, TestFieldCollectionResource, TestReloadAndDeleteDocs, SuggestComponentTest, TestIndexSearcher, TestRecovery, TestUtils, ResourceLoaderTest, TestFieldCacheSanityChecker, TestRandomRequestDistribution, TestSystemIdResolver, ZkStateWriterTest, HdfsRecoveryZkTest, SpellCheckCollatorTest, TestLeaderElectionWithEmptyReplica, TestCryptoKeys, HighlighterMaxOffsetTest, SolrXmlInZkTest, WordBreakSolrSpellCheckerTest, LeaderElectionIntegrationTest, ClusterStateTest, TestCustomDocTransformer, TestFieldTypeCollectionResource, CloudMLTQParserTest, OverseerTaskQueueTest, CurrencyFieldOpenExchangeTest, TestLockTree, ReplicaListTransformerTest, HdfsLockFactoryTest, TestConfigSets, RAMDirectoryFactoryTest, TestSlowCompositeReaderWrapper, TestQuerySenderListener, SyncSliceTest, RecoveryZkTest, ClusterStateUpdateTest, TestRandomFaceting, ZkSolrClientTest, ZkCLITest, TermVectorComponentDistributedTest, TestRealTimeGet, TestStressVersions, SolrCoreTest, SolrCmdDistributorTest, TestFunctionQuery, TestLazyCores, SoftAutoCommitTest, CoreAdminHandlerTest, NoCacheHeaderTest, TestPseudoReturnFields, FieldMutatingUpdateProcessorTest, TestAtomicUpdateErrorCases, StatelessScriptUpdateProcessorFactoryTest, CacheHeaderTest, TestWriterPerf, FileBasedSpellCheckerTest, PathHierarchyTokenizerFactoryTest, FieldAnalysisRequestHandlerTest, IndexSchemaRuntimeFieldTest, IndexSchemaTest, UpdateRequestProcessorFactoryTest, TestPartialUpdateDeduplication, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, TestSolrIndexConfig, CopyFieldTest, SolrIndexConfigTest, BadComponentTest, TestStressRecovery, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestHighlightDedupGrouping, TestSimpleTrackingShardHandler, TestJettySolrRunner, ConnectionReuseTest, ActionThrottleTest, AliasIntegrationTest, CdcrReplicationDistributedZkTest, CollectionReloadTest, CollectionsAPISolrJTest, DeleteStatusTest, DistribJoinFromCollectionTest, DistributedVersionInfoTest, DocValuesNotIndexedTest]
[junit4] Completed [487/680 (1!)] on J1 in 16.13s, 4 tests, 1 failure <<< FAILURES!
[...truncated 63069 lines...]